From 886e3fa22e6b1382dcc0c06aeeefab3ceb6362ec Mon Sep 17 00:00:00 2001
From: MDLeom <2809763-curben@users.noreply.gitlab.com>
Date: Tue, 4 May 2021 08:11:15 +0000
Subject: [PATCH] fix: add oisd exclusion list

- https://oisd.nl/excludes.php
---
 .gitlab-ci.yml  |  2 +-
 script.sh       | 13 +++++++++++--
 src/exclude.txt |  2 +-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 051219f8..82584127 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -9,7 +9,7 @@ build_job:
   stage: build
 
   before_script:
-    - 'which ssh-agent || (apk update && apk add curl openssh-client git grep)'
+    - apk update && apk add curl openssh-client git grep xmlstarlet
     - eval $(ssh-agent -s)
     - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
     - mkdir -p ~/.ssh
diff --git a/script.sh b/script.sh
index c1b42671..3b6807c7 100644
--- a/script.sh
+++ b/script.sh
@@ -11,6 +11,7 @@ cd "tmp/"
 curl -L "https://urlhaus.abuse.ch/downloads/csv/" -o "urlhaus.zip"
 curl -L "https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" -o "top-1m-umbrella.zip"
 curl -L "https://tranco-list.eu/top-1m.csv.zip" -o "top-1m-tranco.zip"
+curl -L "https://oisd.nl/excludes.php" -o "oisd-exclude.html"
 
 cp -f "../src/exclude.txt" "."
 
@@ -81,8 +82,16 @@ grep -F "." | \
 sed "s/^www\.//g" | \
 sort -u > "top-1m-tranco.txt"
 
+## Parse oisd exclusion list
+cat "oisd-exclude.html" | \
+# https://stackoverflow.com/a/47600828
+xmlstarlet format --recover --html 2>/dev/null | \
+xmlstarlet select --html --template --value-of '//a' | \
+## Append new line https://unix.stackexchange.com/a/31955
+sed '$a\' > "oisd-exclude.txt"
+
 # Merge Umbrella and self-maintained top domains
-cat "top-1m-umbrella.txt" "top-1m-tranco.txt" "exclude.txt" | \
+cat "top-1m-umbrella.txt" "top-1m-tranco.txt" "exclude.txt" "oisd-exclude.txt" | \
 sort -u > "top-1m-well-known.txt"
 
 
@@ -374,7 +383,7 @@ sed "2s/Domains Blocklist/Hosts Blocklist (IE)/" > "../urlhaus-filter-online.tpl
 
 
 ## Clean up artifacts
-rm "URLhaus.csv" "top-1m-umbrella.zip" "top-1m-umbrella.txt" "top-1m-tranco.txt"
+rm "URLhaus.csv" "top-1m-umbrella.zip" "top-1m-umbrella.txt" "top-1m-tranco.txt" "oisd-exclude.html" "oisd-exclude.txt"
 
 
 cd ../
diff --git a/src/exclude.txt b/src/exclude.txt
index f53353b6..c065170e 100644
--- a/src/exclude.txt
+++ b/src/exclude.txt
@@ -68,4 +68,4 @@ s3-us-gov-east-1.amazonaws.com
 s3-us-gov-west-1.amazonaws.com
 srv-store4.gofile.io
 srv-store6.gofile.io
-srv-file9.gofile.io
\ No newline at end of file
+srv-file9.gofile.io