diff --git a/README.md b/README.md index 400ea99b..4fcd1eff 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,7 @@ filters: [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/) [OpenPhish](https://openphish.com/): Available [free of charge](https://openphish.com/terms.html) by OpenPhish -[mitchellkrogza/Phishing.Database](https://github.com/mitchellkrogza/Phishing.Database): MIT License +[IPThreat](https://ipthreat.net): CC BY-SA 4.0 [Tranco List](https://tranco-list.eu/): [MIT License](https://choosealicense.com/licenses/mit/) diff --git a/src/script.sh b/src/script.sh index df5176e0..1e61b4bf 100644 --- a/src/script.sh +++ b/src/script.sh @@ -70,6 +70,7 @@ cd "tmp/" ## Prepare datasets curl "https://openphish.com/feed.txt" -o "openphish-raw.txt" +curl "https://lists.ipthreat.net/file/ipthreat-lists/phishing/phishing-threat-0.txt.gz" -o "ipthreat.gz" curl "https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" -o "top-1m-umbrella.zip" curl "https://tranco-list.eu/top-1m.csv.zip" -o "top-1m-tranco.zip" @@ -112,16 +113,18 @@ sed "s/^www\.//g" | \ # url encode space #11 sed "s/ /%20/g" > "openphish.txt" -# https://github.com/mitchellkrogza/Phishing.Database/raw/master/ALL-phishing-links.tar.gz -# tar xzfO "ALL-phishing-links.tar.gz" | \ -# tr "[:upper:]" "[:lower:]" | \ -# cut -f 3- -d "/" | \ -# grep -F "." | \ -# sed "s/^www\.//g" | \ -# sed "s/ /%20/g" > "phishing.db.txt" +gzip -dc "ipthreat.gz" | \ +# remove comment +sed "/^#/d" | \ +sed "s/ # .*//g" | \ +tr "[:upper:]" "[:lower:]" | \ +cut -f 3- -d "/" | \ +grep -F "." | \ +sed "s/^www\.//g" | \ +sed "s/ /%20/g" > "ipthreat.txt" ## Combine all sources -cat "openphish.txt" | \ +cat "openphish.txt" "ipthreat.txt" | \ sort -u > "phishing.txt" ## Parse domain and IP address only @@ -251,7 +254,7 @@ SECOND_LINE="! Updated: $CURRENT_TIME" THIRD_LINE="! Expires: 1 day (update frequency)" FOURTH_LINE="! Homepage: https://gitlab.com/malware-filter/phishing-filter" FIFTH_LINE="! License: https://gitlab.com/malware-filter/phishing-filter#license" -SIXTH_LINE="! Sources: openphish.com, github.com/mitchellkrogza/Phishing.Database" +SIXTH_LINE="! Sources: openphish.com, ipthreat.net" COMMENT_UBO="$FIRST_LINE\n$SECOND_LINE\n$THIRD_LINE\n$FOURTH_LINE\n$FIFTH_LINE\n$SIXTH_LINE" mkdir -p "../public/"