curben
/
phishing-filter
mirror of https://gitlab.com/malware-filter/phishing-filter
1
0
Fork 0
Code Issues Releases Wiki Activity

feat: add ipthreat.net source

This commit is contained in:
MDLeom 2025-02-08 06:24:03 +00:00
parent d7145f19e1
commit 7f90191c49
No known key found for this signature in database
GPG Key ID: 32D3E28E96A695E8
2 changed files with 13 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -259,7 +259,7 @@ filters: [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)
[OpenPhish](https://openphish.com/): Available [free of charge](https://openphish.com/terms.html) by OpenPhish
[mitchellkrogza/Phishing.Database](https://github.com/mitchellkrogza/Phishing.Database): MIT License
[IPThreat](https://ipthreat.net): CC BY-SA 4.0
[Tranco List](https://tranco-list.eu/): [MIT License](https://choosealicense.com/licenses/mit/)

21
src/script.sh
View File

@ -70,6 +70,7 @@ cd "tmp/"
## Prepare datasets
curl "https://openphish.com/feed.txt" -o "openphish-raw.txt"
curl "https://lists.ipthreat.net/file/ipthreat-lists/phishing/phishing-threat-0.txt.gz" -o "ipthreat.gz"
curl "https://s3-us-west-1.amazonaws.com/umbrella-static/top-1m.csv.zip" -o "top-1m-umbrella.zip"
curl "https://tranco-list.eu/top-1m.csv.zip" -o "top-1m-tranco.zip"
@ -112,16 +113,18 @@ sed "s/^www\.//g" | \
# url encode space #11
sed "s/ /%20/g" > "openphish.txt"
# https://github.com/mitchellkrogza/Phishing.Database/raw/master/ALL-phishing-links.tar.gz
# tar xzfO "ALL-phishing-links.tar.gz" | \
# tr "[:upper:]" "[:lower:]" | \
# cut -f 3- -d "/" | \
# grep -F "." | \
# sed "s/^www\.//g" | \
# sed "s/ /%20/g" > "phishing.db.txt"
gzip -dc "ipthreat.gz" | \
# remove comment
sed "/^#/d" | \
sed "s/ # .*//g" | \
tr "[:upper:]" "[:lower:]" | \
cut -f 3- -d "/" | \
grep -F "." | \
sed "s/^www\.//g" | \
sed "s/ /%20/g" > "ipthreat.txt"
## Combine all sources
cat "openphish.txt" | \
cat "openphish.txt" "ipthreat.txt" | \
sort -u > "phishing.txt"
## Parse domain and IP address only
@ -251,7 +254,7 @@ SECOND_LINE="! Updated: $CURRENT_TIME"
THIRD_LINE="! Expires: 1 day (update frequency)"
FOURTH_LINE="! Homepage: https://gitlab.com/malware-filter/phishing-filter"
FIFTH_LINE="! License: https://gitlab.com/malware-filter/phishing-filter#license"
SIXTH_LINE="! Sources: openphish.com, github.com/mitchellkrogza/Phishing.Database"
SIXTH_LINE="! Sources: openphish.com, ipthreat.net"
COMMENT_UBO="$FIRST_LINE\n$SECOND_LINE\n$THIRD_LINE\n$FOURTH_LINE\n$FIFTH_LINE\n$SIXTH_LINE"
mkdir -p "../public/"