urlhaus-filter/utils/malware-domains.sh

#!/bin/sh

set -e -x

## Parse domains from URLhaus excluding popular domains

cat URLhaus.txt | \
# Exclude Umbrella Top 1M and well-known domains
# grep inverse match whole line
grep -Fx -vf urlhaus-top-domains.txt > malware-domains.txt
feat: include full URL for popular domains 2019-05-11 09:19:25 +00:00			`#!/bin/sh`

refactor: move script executions from CI config to index.sh easier to test locally 2019-05-27 05:31:57 +00:00			`set -e -x`

feat: include full URL for popular domains 2019-05-11 09:19:25 +00:00			`## Parse domains from URLhaus excluding popular domains`

fix: use simple URL list we no longer care the status of URL bb817d9838d35eda22c5b30018488be0bc19e0db 2019-05-27 06:29:08 +00:00			`cat URLhaus.txt \| \`
feat: include full URL for popular domains 2019-05-11 09:19:25 +00:00			`# Exclude Umbrella Top 1M and well-known domains`
			`# grep inverse match whole line`
perf: grep using urlhaus-top-domains.txt instead of much larger top-1m.txt 2019-05-12 02:59:52 +00:00			`grep -Fx -vf urlhaus-top-domains.txt > malware-domains.txt`