From d1445dcc4ce99095a2e4c0c6261114fd43a7d0b4 Mon Sep 17 00:00:00 2001
From: MDLeom <2809763-curben@users.noreply.gitlab.com>
Date: Fri, 7 May 2021 11:14:37 +0000
Subject: [PATCH] Revert "fix: add oisd exclusion list"

This reverts commit 886e3fa22e6b1382dcc0c06aeeefab3ceb6362ec.
- build failed for unknown reason
---
 .gitlab-ci.yml  |  2 +-
 script.sh       | 13 ++-----------
 src/exclude.txt |  2 +-
 3 files changed, 4 insertions(+), 13 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 82584127..051219f8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,7 +9,7 @@ build_job:
   stage: build
 
   before_script:
-    - apk update && apk add curl openssh-client git grep xmlstarlet
+    - 'which ssh-agent || (apk update && apk add curl openssh-client git grep)'
     - eval $(ssh-agent -s)
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
     - mkdir -p ~/.ssh
diff --git a/script.sh b/script.sh
index 3b6807c7..c1b42671 100644
--- a/script.sh
+++ b/script.sh
@@ -11,7 +11,6 @@ cd "tmp/"
 curl -L "https://urlhaus.abuse.ch/downloads/csv/" -o "urlhaus.zip"
 curl -L "https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" -o "top-1m-umbrella.zip"
 curl -L "https://tranco-list.eu/top-1m.csv.zip" -o "top-1m-tranco.zip"
-curl -L "https://oisd.nl/excludes.php" -o "oisd-exclude.html"
 
 cp -f "../src/exclude.txt" "."
 
@@ -82,16 +81,8 @@ grep -F "." | \
 sed "s/^www\.//g" | \
 sort -u > "top-1m-tranco.txt"
 
-## Parse oisd exclusion list
-cat "oisd-exclude.html" | \
-# https://stackoverflow.com/a/47600828
-xmlstarlet format --recover --html 2>/dev/null | \
-xmlstarlet select --html --template --value-of '//a' | \
-## Append new line https://unix.stackexchange.com/a/31955
-sed '$a\' > "oisd-exclude.txt"
-
 # Merge Umbrella and self-maintained top domains
-cat "top-1m-umbrella.txt" "top-1m-tranco.txt" "exclude.txt" "oisd-exclude.txt" | \
+cat "top-1m-umbrella.txt" "top-1m-tranco.txt" "exclude.txt" | \
 sort -u > "top-1m-well-known.txt"
 
 
@@ -383,7 +374,7 @@ sed "2s/Domains Blocklist/Hosts Blocklist (IE)/" > "../urlhaus-filter-online.tpl
 
 
 ## Clean up artifacts
-rm "URLhaus.csv" "top-1m-umbrella.zip" "top-1m-umbrella.txt" "top-1m-tranco.txt" "oisd-exclude.html" "oisd-exclude.txt"
+rm "URLhaus.csv" "top-1m-umbrella.zip" "top-1m-umbrella.txt" "top-1m-tranco.txt"
 
 
 cd ../
diff --git a/src/exclude.txt b/src/exclude.txt
index c065170e..f53353b6 100644
--- a/src/exclude.txt
+++ b/src/exclude.txt
@@ -68,4 +68,4 @@ s3-us-gov-east-1.amazonaws.com
 s3-us-gov-west-1.amazonaws.com
 srv-store4.gofile.io
 srv-store6.gofile.io
-srv-file9.gofile.io
+srv-file9.gofile.io
\ No newline at end of file