feat: use Tranco list

- replace Cisco Umbrella
- Tranco's ranking method is open-source
  * https://github.com/DistriNet/tranco-list
  * https://tranco-list.eu/
This commit is contained in:
curben 2020-04-01 11:07:54 +01:00
parent 6f16c016c5
commit bb9e8ca7cf
No known key found for this signature in database
GPG Key ID: 5D9DB57A25D34EE3
1 changed files with 2 additions and 2 deletions

View File

@ -8,7 +8,7 @@ mkdir -p "tmp/" && cd "tmp/"
## Prepare datasets
wget "https://urlhaus.abuse.ch/downloads/csv/" -O "urlhaus.zip"
wget "https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" -O "top-1m.csv.zip"
wget "https://tranco-list.eu/top-1m.csv.zip" -O "top-1m.csv.zip"
cp "../src/exclude.txt" "."
@ -49,7 +49,7 @@ cut -f 1 -d ":" | \
sort -u > "urlhaus-domains-online.txt"
## Parse the Cisco Umbrella 1 Million
## Parse the Tranco 1 Million
unzip -p "top-1m.csv.zip" | \
dos2unix | \
# Parse domains only