urlhaus-filter/.gitlab-ci.yml

72 lines
1.8 KiB
YAML

image: alpine:latest # Use the latest version of Alpine Linux docker image
before_script:
# Install dependencies
- 'which ssh-agent || ( apk update && apk add openssh-client git)'
# Run ssh-agent
- eval $(ssh-agent -s)
# Import private key with ed25519 workaround
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
# Create the SSH directory and give it the right permissions
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
# Add gitlab.com as ssh known host
- ssh-keyscan gitlab.com >> ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
# Set commit author
- git config --global user.name "curben-bot"
- git config --global user.email "curben-bot@users.noreply.gitlab.com"
deploy:
script:
# SSH to gitlab.com
- ssh git@gitlab.com
# Shallow cloning for faster cloning
- git clone --depth 3 git@gitlab.com:curben/urlhaus-filter.git build
# Change to the downloaded repo directory
- cd build/
# Create a temporary working folder
- mkdir tmp/ && cd tmp/
# Download URLhaus database and Umbrella Top 1M
- sh ../utils/prerequisites.sh
# Process the Umbrella Top 1M
- sh ../utils/umbrella-top-1m.sh
# Parse popular domains that also appear in URLhaus
- sh ../utils/urlhaus-top-domains.sh
# Parse domains from URLhaus excluding popular domains
- sh ../utils/malware-domains.sh
# Parse malware URLs from popular domains
- sh ../utils/malware-url-top-domains.sh
# Merge malware domains and URLs
- sh ../utils/urlhaus-filter.sh
# Commit the changes
- sh ../utils/commit.sh
# Clean up the working folder
- cd ../ && rm -r tmp/
# Push the commit
- git push
only:
# Allow CI to be triggered by schedule
- schedules
# Allow CI to be triggered manually
- web