diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index 0c8795d2..c3ba21f2 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -14,7 +14,7 @@ jobs: - name: Install Dependencies run: | apk update - apk add brotli curl git grep jq xmlstarlet zstd + apk add brotli curl git grep jq zstd - name: Build env: PHISHTANK_API: ${{ secrets.PHISHTANK_API }} diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 82bccf2d..f261035c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,7 +12,7 @@ build_job: stage: build before_script: - - apk update && apk add brotli curl grep jq xmlstarlet zstd + - apk update && apk add brotli curl grep jq zstd script: - sh src/script.sh diff --git a/src/script.sh b/src/script.sh index 1e61b4bf..44c77224 100644 --- a/src/script.sh +++ b/src/script.sh @@ -160,20 +160,6 @@ grep -F "." | \ sed "s/^www\.//g" | \ sort -u > "top-1m-tranco.txt" -# ## Parse oisd exclusion list -# cat "oisd-exclude.html" | \ -# # https://stackoverflow.com/a/47600828 -# xmlstarlet format --recover --html 2>/dev/null | \ -# xmlstarlet select --html --template --value-of '//a' | \ -# ## Append new line https://unix.stackexchange.com/a/31955 -# sed '$a\' > "oisd-exclude.txt" - -# # html-xml-utils -# cat "oisd-exclude.html" | \ -# hxwls | \ -# grep -F '?w=' | \ -# sed 's/^?w=//g' > "oisd-exclude.txt" - # Merge Umbrella, Tranco, Radar and self-maintained top domains cat "top-1m-umbrella.txt" "top-1m-tranco.txt" "exclude.txt" | \ sort -u > "top-1m-well-known.txt"