2019-05-11 09:19:25 +00:00
|
|
|
#!/bin/sh
|
|
|
|
|
|
2019-05-27 05:31:57 +00:00
|
|
|
set -e -x
|
|
|
|
|
|
2019-05-11 09:19:25 +00:00
|
|
|
# Download URLhaus database
|
2019-05-27 06:29:08 +00:00
|
|
|
wget https://urlhaus.abuse.ch/downloads/text/ -O ../src/URLhaus.txt
|
2019-05-11 09:19:25 +00:00
|
|
|
|
|
|
|
|
# Download Cisco Umbrella 1 Million
|
|
|
|
|
wget https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip -O top-1m.csv.zip
|
|
|
|
|
|
|
|
|
|
cp ../src/exclude.txt .
|
2019-05-27 06:29:08 +00:00
|
|
|
|
|
|
|
|
## Clean up URLhaus.txt
|
|
|
|
|
cat ../src/URLhaus.txt | \
|
|
|
|
|
# Convert DOS to Unix line ending
|
|
|
|
|
dos2unix | \
|
2019-05-28 00:29:02 +00:00
|
|
|
# Remove comment
|
2019-05-28 01:03:07 +00:00
|
|
|
sed 's/^#.*//g' | \
|
2019-05-27 06:29:08 +00:00
|
|
|
# Remove http(s)://
|
|
|
|
|
cut -f 3 -d '/' | \
|
2019-05-28 00:54:04 +00:00
|
|
|
# Remove port number
|
|
|
|
|
cut -f 1 -d ':' | \
|
2019-05-27 06:29:08 +00:00
|
|
|
# Remove www
|
|
|
|
|
# Only matches domains that start with www
|
|
|
|
|
# Not examplewww.com
|
|
|
|
|
sed 's/^www\.//g' | \
|
|
|
|
|
# Sort and remove duplicates
|
2019-05-28 00:29:02 +00:00
|
|
|
sort -u > urlhaus.txt
|
