urlhaus-filter/script.sh

361 lines
12 KiB
Bash
Raw Normal View History

#!/bin/sh
set -efux -o pipefail
## Create a temporary working folder
mkdir -p "tmp/"
cd "tmp/"
## Prepare datasets
curl -L "https://urlhaus.abuse.ch/downloads/csv/" -o "urlhaus.zip"
curl -L "https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" -o "top-1m-umbrella.zip"
curl -L "https://tranco-list.eu/top-1m.csv.zip" -o "top-1m-tranco.zip"
cp "../src/exclude.txt" "."
2020-04-01 09:52:12 +00:00
## Prepare URLhaus.csv
unzip -p "urlhaus.zip" | \
# Convert DOS to Unix line ending
dos2unix | \
2021-03-18 08:43:53 +00:00
tr "[:upper:]" "[:lower:]" | \
# Remove comment
sed "/^#/d" > "URLhaus.csv"
2020-04-01 09:52:12 +00:00
## Parse URLs
cat "URLhaus.csv" | \
cut -f 6 -d '"' | \
cut -f 3- -d "/" | \
2019-08-04 07:54:07 +00:00
# Domain must have at least a 'dot'
grep -F "." | \
# Remove invalid protocol, see #32
sed -E "s/^(ttps:\/\/|https:\/|http\/)//g" | \
# Remove www.
sed "s/^www\.//g" | \
sort -u > "urlhaus.txt"
## Parse domain and IP address only
cat "urlhaus.txt" | \
cut -f 1 -d "/" | \
cut -f 1 -d ":" | \
2020-05-15 23:28:22 +00:00
# Remove invalid domains, see #15
grep -vF "??" | \
cut -f 1 -d "?" | \
sort -u > "urlhaus-domains.txt"
2020-04-01 09:52:12 +00:00
## Parse online URLs only
cat "URLhaus.csv" | \
grep '"online"' | \
cut -f 6 -d '"' | \
cut -f 3- -d "/" | \
sed "s/^www\.//g" | \
sort -u > "urlhaus-online.txt"
cat "urlhaus-online.txt" | \
cut -f 1 -d "/" | \
cut -f 1 -d ":" | \
2020-05-15 23:28:22 +00:00
grep -vF "??" | \
cut -f 1 -d "?" | \
sort -u > "urlhaus-domains-online.txt"
## Parse the Umbrella 1 Million
unzip -p "top-1m-umbrella.zip" | \
dos2unix | \
2021-03-18 08:43:53 +00:00
tr "[:upper:]" "[:lower:]" | \
# Parse domains only
cut -f 2 -d "," | \
grep -F "." | \
# Remove www.
sed "s/^www\.//g" | \
sort -u > "top-1m-umbrella.txt"
## Parse the Tranco 1 Million
unzip -p "top-1m-tranco.zip" | \
dos2unix | \
2021-03-18 08:43:53 +00:00
tr "[:upper:]" "[:lower:]" | \
# Parse domains only
cut -f 2 -d "," | \
grep -F "." | \
# Remove www.
sed "s/^www\.//g" | \
sort -u > "top-1m-tranco.txt"
# Merge Umbrella and self-maintained top domains
cat "top-1m-umbrella.txt" "top-1m-tranco.txt" "exclude.txt" | \
sort -u > "top-1m-well-known.txt"
## Parse popular domains from URLhaus
cat "urlhaus-domains.txt" | \
# grep match whole line
grep -Fx -f "top-1m-well-known.txt" > "urlhaus-top-domains.txt"
## Parse domains from URLhaus excluding popular domains
cat "urlhaus-domains.txt" | \
grep -F -vf "urlhaus-top-domains.txt" > "malware-domains.txt"
cat "urlhaus-domains-online.txt" | \
grep -F -vf "urlhaus-top-domains.txt" > "malware-domains-online.txt"
2019-05-29 06:04:31 +00:00
## Parse malware URLs from popular domains
cat "urlhaus.txt" | \
grep -F -f "urlhaus-top-domains.txt" | \
sed "s/^/||/g" | \
sed "s/$/\$all/g" > "malware-url-top-domains.txt"
cat "urlhaus-online.txt" | \
grep -F -f "urlhaus-top-domains.txt" | \
sed "s/^/||/g" | \
sed "s/$/\$all/g" > "malware-url-top-domains-online.txt"
2021-03-18 08:43:53 +00:00
cat "urlhaus-online.txt" | \
grep -F -f "urlhaus-top-domains.txt" > "malware-url-top-domains-raw-online.txt"
## Merge malware domains and URLs
CURRENT_TIME="$(date -R -u)"
FIRST_LINE="! Title: Malicious URL Blocklist"
SECOND_LINE="! Updated: $CURRENT_TIME"
THIRD_LINE="! Expires: 1 day (update frequency)"
FOURTH_LINE="! Homepage: https://gitlab.com/curben/urlhaus-filter"
FIFTH_LINE="! License: https://gitlab.com/curben/urlhaus-filter#license"
SIXTH_LINE="! Source: https://urlhaus.abuse.ch/api/"
COMMENT_ABP="$FIRST_LINE\n$SECOND_LINE\n$THIRD_LINE\n$FOURTH_LINE\n$FIFTH_LINE\n$SIXTH_LINE"
2020-10-04 02:39:27 +00:00
cat "malware-domains.txt" "malware-url-top-domains.txt" | \
sort | \
sed '1 i\'"$COMMENT_ABP"'' > "../urlhaus-filter.txt"
cat "malware-domains-online.txt" "malware-url-top-domains-online.txt" | \
sort | \
sed '1 i\'"$COMMENT_ABP"'' | \
sed "1s/Malicious/Online Malicious/" > "../urlhaus-filter-online.txt"
# Adguard Home (#19, #22)
cat "malware-domains.txt" | \
sed "s/^/||/g" | \
sed "s/$/^/g" > "malware-domains-adguard-home.txt"
cat "malware-domains-online.txt" | \
sed "s/^/||/g" | \
sed "s/$/^/g" > "malware-domains-online-adguard-home.txt"
cat "malware-domains-adguard-home.txt" | \
sort | \
sed '1 i\'"$COMMENT_ABP"'' | \
sed "1s/Blocklist/Blocklist (AdGuard Home)/" > "../urlhaus-filter-agh.txt"
cat "malware-domains-online-adguard-home.txt" | \
sort | \
sed '1 i\'"$COMMENT_ABP"'' | \
sed "1s/Malicious/Online Malicious/" | \
sed "1s/Blocklist/Blocklist (AdGuard Home)/" > "../urlhaus-filter-agh-online.txt"
# Adguard browser extension
cat "malware-domains.txt" | \
sed "s/^/||/g" | \
sed "s/$/\$all/g" > "malware-domains-adguard.txt"
cat "malware-domains-online.txt" | \
sed "s/^/||/g" | \
sed "s/$/\$all/g" > "malware-domains-online-adguard.txt"
cat "malware-domains-adguard.txt" "malware-url-top-domains.txt" | \
sort | \
sed '1 i\'"$COMMENT_ABP"'' | \
sed "1s/Blocklist/Blocklist (AdGuard)/" > "../urlhaus-filter-ag.txt"
cat "malware-domains-online-adguard.txt" "malware-url-top-domains-online.txt" | \
sort | \
sed '1 i\'"$COMMENT_ABP"'' | \
sed "1s/Malicious/Online Malicious/" | \
sed "1s/Blocklist/Blocklist (AdGuard)/" > "../urlhaus-filter-ag-online.txt"
# Vivaldi
cat "malware-domains.txt" | \
sed "s/^/||/g" | \
sed "s/$/\$document/g" > "malware-domains-vivaldi.txt"
cat "malware-domains-online.txt" | \
sed "s/^/||/g" | \
sed "s/$/\$document/g" > "malware-domains-online-vivaldi.txt"
cat "malware-domains-vivaldi.txt" "malware-url-top-domains.txt" | \
sed "s/\$all$/\$document/g" | \
sort | \
sed '1 i\'"$COMMENT_ABP"'' | \
sed "1s/Blocklist/Blocklist (Vivaldi)/" > "../urlhaus-filter-vivaldi.txt"
cat "malware-domains-online-vivaldi.txt" "malware-url-top-domains-online.txt" | \
sed "s/\$all$/\$document/g" | \
sort | \
sed '1 i\'"$COMMENT_ABP"'' | \
sed "1s/Malicious/Online Malicious/" | \
sed "1s/Blocklist/Blocklist (Vivaldi)/" > "../urlhaus-filter-vivaldi-online.txt"
2020-04-19 05:19:05 +00:00
## Domains-only blocklist
2020-05-13 10:37:40 +00:00
# awk + head is a workaround for sed prepend
COMMENT=$(printf "$COMMENT_ABP" | sed "s/^!/#/g" | sed "1s/URL/Domains/" | awk '{printf "%s\\n", $0}' | head -c -2)
COMMENT_ONLINE=$(printf "$COMMENT" | sed "1s/Malicious/Online Malicious/" | awk '{printf "%s\\n", $0}' | head -c -2)
2019-10-01 06:58:46 +00:00
cat "malware-domains.txt" | \
2019-10-01 06:58:46 +00:00
sort | \
2020-04-19 05:19:05 +00:00
sed '1 i\'"$COMMENT"'' > "../urlhaus-filter-domains.txt"
2019-10-01 06:58:46 +00:00
cat "malware-domains-online.txt" | \
2019-10-01 06:58:46 +00:00
sort | \
sed '1 i\'"$COMMENT_ONLINE"'' > "../urlhaus-filter-domains-online.txt"
2019-10-01 06:58:46 +00:00
2020-04-19 05:19:05 +00:00
## Hosts file blocklist
cat "../urlhaus-filter-domains.txt" | \
# Exclude comment with #
grep -vE "^#" | \
# Remove IPv4 address
grep -vE "([0-9]{1,3}[\.]){3}[0-9]{1,3}" | \
sed "s/^/0.0.0.0 /g" | \
# Re-insert comment
sed '1 i\'"$COMMENT"'' | \
2020-04-19 05:19:05 +00:00
sed "1s/Domains/Hosts/" > "../urlhaus-filter-hosts.txt"
2020-04-19 05:19:05 +00:00
cat "../urlhaus-filter-domains-online.txt" | \
grep -vE "^#" | \
grep -vE "([0-9]{1,3}[\.]){3}[0-9]{1,3}" | \
sed "s/^/0.0.0.0 /g" | \
sed '1 i\'"$COMMENT_ONLINE"'' | \
sed "1s/Domains/Hosts/" > "../urlhaus-filter-hosts-online.txt"
2020-04-19 05:19:05 +00:00
## Dnsmasq-compatible blocklist
cat "../urlhaus-filter-hosts.txt" | \
grep -vE "^#" | \
sed "s/^0.0.0.0 /address=\//g" | \
sed "s/$/\/0.0.0.0/g" | \
sed '1 i\'"$COMMENT"'' | \
2020-04-19 05:19:05 +00:00
sed "1s/Blocklist/dnsmasq Blocklist/" > "../urlhaus-filter-dnsmasq.conf"
cat "../urlhaus-filter-hosts-online.txt" | \
grep -vE "^#" | \
sed "s/^0.0.0.0 /address=\//g" | \
sed "s/$/\/0.0.0.0/g" | \
sed '1 i\'"$COMMENT_ONLINE"'' | \
2020-04-19 05:19:05 +00:00
sed "1s/Blocklist/dnsmasq Blocklist/" > "../urlhaus-filter-dnsmasq-online.conf"
2020-04-19 08:02:03 +00:00
## BIND-compatible blocklist
cat "../urlhaus-filter-hosts.txt" | \
grep -vE "^#" | \
sed 's/^0.0.0.0 /zone "/g' | \
sed 's/$/" { type master; notify no; file "null.zone.file"; };/g' | \
2020-04-19 08:02:03 +00:00
sed '1 i\'"$COMMENT"'' | \
sed "1s/Blocklist/BIND Blocklist/" > "../urlhaus-filter-bind.conf"
cat "../urlhaus-filter-hosts-online.txt" | \
grep -vE "^#" | \
sed 's/^0.0.0.0 /zone "/g' | \
sed 's/$/" { type master; notify no; file "null.zone.file"; };/g' | \
sed '1 i\'"$COMMENT_ONLINE"'' | \
2020-04-19 08:02:03 +00:00
sed "1s/Blocklist/BIND Blocklist/" > "../urlhaus-filter-bind-online.conf"
2020-04-19 08:42:25 +00:00
## Unbound-compatible blocklist
cat "../urlhaus-filter-hosts.txt" | \
grep -vE "^#" | \
sed 's/^0.0.0.0 /local-zone: "/g' | \
sed 's/$/" always_nxdomain/g' | \
sed '1 i\'"$COMMENT"'' | \
sed "1s/Blocklist/Unbound Blocklist/" > "../urlhaus-filter-unbound.conf"
cat "../urlhaus-filter-hosts-online.txt" | \
grep -vE "^#" | \
sed 's/^0.0.0.0 /local-zone: "/g' | \
sed 's/$/" always_nxdomain/g' | \
sed '1 i\'"$COMMENT_ONLINE"'' | \
2020-04-19 08:42:25 +00:00
sed "1s/Blocklist/Unbound Blocklist/" > "../urlhaus-filter-unbound-online.conf"
2021-03-18 10:18:59 +00:00
## Temporarily disable command print
set +x
# Snort & Suricata
2021-03-19 18:41:49 +00:00
rm -f "../urlhaus-filter-snort2-online.rules" \
"../urlhaus-filter-snort3-online.rules" \
"../urlhaus-filter-suricata-online.rules"
2021-03-18 10:18:59 +00:00
SID="100000001"
while read DOMAIN; do
SN_RULE="alert tcp \$HOME_NET any -> \$EXTERNAL_NET [80,443] (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; content:\"GET\"; http_method; content:\"$DOMAIN\"; content:\"Host\"; http_header; classtype:trojan-activity; sid:$SID; rev:1;)"
2021-03-19 18:41:49 +00:00
SN3_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; http_header:field host; content:\"$DOMAIN\",nocase; classtype:trojan-activity; sid:$SID; rev:1;)"
2021-03-18 10:18:59 +00:00
SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.host; content:\"$DOMAIN\"; classtype:trojan-activity; sid:$SID; rev:1;)"
echo "$SN_RULE" >> "../urlhaus-filter-snort2-online.rules"
2021-03-19 18:41:49 +00:00
echo "$SN3_RULE" >> "../urlhaus-filter-snort3-online.rules"
2021-03-18 10:18:59 +00:00
echo "$SR_RULE" >> "../urlhaus-filter-suricata-online.rules"
SID=$(( $SID + 1 ))
done < "malware-domains-online.txt"
while read URL; do
HOST=$(echo "$URL" | cut -d"/" -f1)
URI=$(echo "$URL" | sed -e "s/^$HOST//" -e "s/;/\\\;/g")
2021-03-18 10:18:59 +00:00
2021-03-19 22:16:30 +00:00
# Snort2 only supports <=2047 characters of `content`
SN_RULE="alert tcp \$HOME_NET any -> \$EXTERNAL_NET [80,443] (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; content:\"GET\"; http_method; content:\"$(echo $URI | cut -c -2047)\"; http_uri; nocase; content:\"$HOST\"; content:\"Host\"; http_header; classtype:trojan-activity; sid:$SID; rev:1;)"
2021-03-18 10:18:59 +00:00
2021-03-19 18:41:49 +00:00
SN3_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; http_header:field host; content:\"$HOST\",nocase; http_uri; content:\"$URI\",nocase; classtype:trojan-activity; sid:$SID; rev:1;)"
2021-03-18 10:18:59 +00:00
SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"urlhaus-filter malicious website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.uri; content:\"$URI\"; endswith; nocase; http.host; content:\"$HOST\"; classtype:trojan-activity; sid:$SID; rev:1;)"
echo "$SN_RULE" >> "../urlhaus-filter-snort2-online.rules"
2021-03-19 18:41:49 +00:00
echo "$SN3_RULE" >> "../urlhaus-filter-snort3-online.rules"
2021-03-18 10:18:59 +00:00
echo "$SR_RULE" >> "../urlhaus-filter-suricata-online.rules"
SID=$(( $SID + 1 ))
done < "malware-url-top-domains-raw-online.txt"
## Re-enable command print
set -x
2021-03-19 22:15:17 +00:00
sed -i '1 i\'"$COMMENT_ONLINE"'' "../urlhaus-filter-snort2-online.rules"
sed -i "1s/Domains Blocklist/URL Snort2 Ruleset/" "../urlhaus-filter-snort2-online.rules"
2021-03-18 10:18:59 +00:00
2021-03-19 22:15:17 +00:00
sed -i '1 i\'"$COMMENT_ONLINE"'' "../urlhaus-filter-snort3-online.rules"
sed -i "1s/Domains Blocklist/URL Snort3 Ruleset/" "../urlhaus-filter-snort3-online.rules"
2021-03-19 18:41:49 +00:00
2021-03-19 22:15:17 +00:00
sed -i '1 i\'"$COMMENT_ONLINE"'' "../urlhaus-filter-suricata-online.rules"
sed -i "1s/Domains Blocklist/URL Suricata Ruleset/" "../urlhaus-filter-suricata-online.rules"
2021-03-18 10:18:59 +00:00
## IE blocklist
COMMENT_IE="msFilterList\n$COMMENT\n: Expires=1\n#"
COMMENT_ONLINE_IE="msFilterList\n$COMMENT_ONLINE\n: Expires=1\n#"
cat "../urlhaus-filter-hosts.txt" | \
grep -vE "^#" | \
sed "s/^0\.0\.0\.0/-d/g" | \
sed '1 i\'"$COMMENT_IE"'' | \
sed "2s/Domains Blocklist/Hosts Blocklist (IE)/" > "../urlhaus-filter.tpl"
cat "../urlhaus-filter-hosts-online.txt" | \
grep -vE "^#" | \
sed "s/^0\.0\.0\.0/-d/g" | \
sed '1 i\'"$COMMENT_ONLINE_IE"'' | \
sed "2s/Domains Blocklist/Hosts Blocklist (IE)/" > "../urlhaus-filter-online.tpl"
## Clean up artifacts
rm "URLhaus.csv" "top-1m-umbrella.zip" "top-1m-umbrella.txt" "top-1m-tranco.txt"
cd ../