urlhaus-filter/.gitlab-ci.yml

image: alpine:latest # Use the latest version of Alpine Linux docker image

before_script:
  # Install dependencies
  - 'which ssh-agent || ( apk update && apk add openssh-client git)'

  # Run ssh-agent
  - eval $(ssh-agent -s)

  # Import private key with ed25519 workaround
  - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null

  # Create the SSH directory and give it the right permissions
  - mkdir -p ~/.ssh
  - chmod 700 ~/.ssh

  # Add gitlab.com as ssh known host
  - ssh-keyscan gitlab.com >> ~/.ssh/known_hosts
  - chmod 644 ~/.ssh/known_hosts

  # Set commit author
  - git config --global user.name "curben-bot"
  - git config --global user.email "curben-bot@users.noreply.gitlab.com"

deploy:
  script:
    # SSH to gitlab.com
    - ssh git@gitlab.com

    # Shallow cloning for faster cloning
    - git clone --depth 3 git@gitlab.com:curben/urlhaus-filter.git build

    # Change to the downloaded repo directory
    - cd build/

    # Create a temporary working folder
    - mkdir tmp/ && cd tmp/

    # Download URLhaus database and Umbrella Top 1M
    - sh ../utils/prerequisites.sh

    # Process the Umbrella Top 1M
    - sh ../utils/umbrella-top-1m.sh

    # Parse popular domains that also appear in URLhaus
    - sh ../utils/urlhaus-top-domains.sh

    # Parse domains from URLhaus excluding popular domains
    - sh ../utils/malware-domains.sh

    # Parse malware URLs from popular domains
    - sh ../utils/malware-url-top-domains.sh

    # Merge malware domains and URLs
    - sh ../utils/urlhaus-filter.sh

    # Commit the changes
    - sh ../utils/commit.sh

    # Clean up the working folder
    - cd ../ && rm -r tmp/

    # Push the commit
    - git push

  only:
    # Allow CI to be triggered by schedule
    - schedules

    # Allow CI to be triggered manually
    - web