curben
/
pup-filter
mirror of https://gitlab.com/malware-filter/pup-filter.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: remove oisd exclusion list 

- captcha
This commit is contained in:
Ming Di Leom 2022-03-28 10:30:48 +00:00
parent 672cd007ca
commit e85e9e5b46
No known key found for this signature in database
GPG Key ID: 32D3E28E96A695E8
1 changed files with 11 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
src/script.sh
View File

@ -4,6 +4,7 @@
set -efux -o pipefail
alias rm="rm -f"
## Use GNU grep, busybox grep is too slow
. "/etc/os-release"
@ -26,7 +27,6 @@ cd "tmp/"
curl -L "https://zhouhanc.github.io/malware-discoverer/blocklist.csv.zip" -o "source.zip"
curl -L "https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" -o "top-1m-umbrella.zip"
curl -L "https://tranco-list.eu/top-1m.csv.zip" -o "top-1m-tranco.zip"
curl -L "https://oisd.nl/excludes.php" -o "oisd-exclude.html"
## Parse URLs
unzip -p "source.zip" | \
@ -60,16 +60,16 @@ sort -u > "top-1m-tranco.txt"
cp "../src/exclude.txt" "."
## Parse oisd exclusion list
cat "oisd-exclude.html" | \
# https://stackoverflow.com/a/47600828
xmlstarlet format --recover --html 2>/dev/null | \
xmlstarlet select --html --template --value-of '//a' | \
## Append new line https://unix.stackexchange.com/a/31955
sed '$a\' > "oisd-exclude.txt"
# ## Parse oisd exclusion list
# cat "oisd-exclude.html" | \
# # https://stackoverflow.com/a/47600828
# xmlstarlet format --recover --html 2>/dev/null | \
# xmlstarlet select --html --template --value-of '//a' | \
# ## Append new line https://unix.stackexchange.com/a/31955
# sed '$a\' > "oisd-exclude.txt"
# Merge Umbrella, Traco and self-maintained top domains
cat "top-1m-umbrella.txt" "top-1m-tranco.txt" "exclude.txt" "oisd-exclude.txt" | \
cat "top-1m-umbrella.txt" "top-1m-tranco.txt" "exclude.txt" | \
sort -u > "top-1m-well-known.txt"
@ -201,7 +201,7 @@ sed "2s/Domains Blocklist/Hosts Blocklist (IE)/" > "../public/pup-filter.tpl"
set +x
## Snort & Suricata rulesets
rm -f "../public/pup-filter-snort2.rules" \
rm "../public/pup-filter-snort2.rules" \
"../public/pup-filter-snort3.rules" \
"../public/pup-filter-suricata.rules"
@ -234,7 +234,7 @@ sed -i "1s/Blocklist/Suricata Ruleset/" "../public/pup-filter-suricata.rules"
## Clean up artifacts
rm "source.zip" "source-domains.txt" "top-1m-umbrella.zip" "top-1m-umbrella.txt" "top-1m-tranco.txt" "oisd-exclude.html" "oisd-exclude.txt"
rm "source.zip" "source-domains.txt" "top-1m-umbrella.zip" "top-1m-umbrella.txt" "top-1m-tranco.txt"
cd ../