image: alpine:latest # Use the latest version of Alpine Linux docker image before_script: # Install dependencies - 'which ssh-agent || ( apk update && apk add openssh-client git)' # Run ssh-agent - eval $(ssh-agent -s) # Import private key with ed25519 workaround - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null # Create the SSH directory and give it the right permissions - mkdir -p ~/.ssh - chmod 700 ~/.ssh # Add gitlab.com as ssh known host - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts - chmod 644 ~/.ssh/known_hosts # Set commit author - git config --global user.name "curben-bot" - git config --global user.email "curben-bot@users.noreply.gitlab.com" deploy: script: # SSH to gitlab.com - ssh git@gitlab.com # Shallow cloning for faster cloning - git clone --depth 3 git@gitlab.com:curben/urlhaus-filter.git build # Change to the downloaded repo directory - cd build/ # Create a temporary working folder - mkdir tmp/ && cd tmp/ # Download URLhaus database and Umbrella Top 1M - sh ../utils/prerequisites.sh # Process the Umbrella Top 1M - sh ../utils/umbrella-top-1m.sh # Parse popular domains that also appear in URLhaus - sh ../utils/urlhaus-top-domains.sh # Parse domains from URLhaus excluding popular domains - sh ../utils/malware-domains.sh # Parse malware URLs from popular domains - sh ../utils/malware-url-top-domains.sh # Merge malware domains and URLs - sh ../utils/urlhaus-filter.sh # Commit the changes - sh ../utils/commit.sh # Clean up the working folder - cd ../ && rm -r tmp/ # Push the commit - git push only: # Allow CI to be triggered by schedule - schedules # Allow CI to be triggered manually - web