diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 82584127..051219f8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,7 @@ build_job: stage: build before_script: - - apk update && apk add curl openssh-client git grep xmlstarlet + - 'which ssh-agent || (apk update && apk add curl openssh-client git grep)' - eval $(ssh-agent -s) - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null - mkdir -p ~/.ssh diff --git a/script.sh b/script.sh index 3b6807c7..c1b42671 100644 --- a/script.sh +++ b/script.sh @@ -11,7 +11,6 @@ cd "tmp/" curl -L "https://urlhaus.abuse.ch/downloads/csv/" -o "urlhaus.zip" curl -L "https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" -o "top-1m-umbrella.zip" curl -L "https://tranco-list.eu/top-1m.csv.zip" -o "top-1m-tranco.zip" -curl -L "https://oisd.nl/excludes.php" -o "oisd-exclude.html" cp -f "../src/exclude.txt" "." @@ -82,16 +81,8 @@ grep -F "." | \ sed "s/^www\.//g" | \ sort -u > "top-1m-tranco.txt" -## Parse oisd exclusion list -cat "oisd-exclude.html" | \ -# https://stackoverflow.com/a/47600828 -xmlstarlet format --recover --html 2>/dev/null | \ -xmlstarlet select --html --template --value-of '//a' | \ -## Append new line https://unix.stackexchange.com/a/31955 -sed '$a\' > "oisd-exclude.txt" - # Merge Umbrella and self-maintained top domains -cat "top-1m-umbrella.txt" "top-1m-tranco.txt" "exclude.txt" "oisd-exclude.txt" | \ +cat "top-1m-umbrella.txt" "top-1m-tranco.txt" "exclude.txt" | \ sort -u > "top-1m-well-known.txt" @@ -383,7 +374,7 @@ sed "2s/Domains Blocklist/Hosts Blocklist (IE)/" > "../urlhaus-filter-online.tpl ## Clean up artifacts -rm "URLhaus.csv" "top-1m-umbrella.zip" "top-1m-umbrella.txt" "top-1m-tranco.txt" "oisd-exclude.html" "oisd-exclude.txt" +rm "URLhaus.csv" "top-1m-umbrella.zip" "top-1m-umbrella.txt" "top-1m-tranco.txt" cd ../ diff --git a/src/exclude.txt b/src/exclude.txt index c065170e..f53353b6 100644 --- a/src/exclude.txt +++ b/src/exclude.txt @@ -68,4 +68,4 @@ s3-us-gov-east-1.amazonaws.com s3-us-gov-west-1.amazonaws.com srv-store4.gofile.io srv-store6.gofile.io -srv-file9.gofile.io +srv-file9.gofile.io \ No newline at end of file