curben
/
phishing-filter
mirror of https://gitlab.com/malware-filter/phishing-filter
1
0
Fork 0
Code Issues Releases Wiki Activity

refactor: deploy filters to gitlab pages 

- 8c94ddba40
This commit is contained in:
MDLeom 2022-01-08 02:55:16 +00:00
parent 12a1c46f1b
commit 6cb2cbff6e
25 changed files with 112 additions and 76425 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

88
.gitlab-ci.yml
View File

@ -1,93 +1,35 @@
stages:
- test
- deploy
- failed_stage
image: alpine:latest # Use the latest version of Alpine Linux docker image
image: alpine:latest
build_job:
stage: test
stage: build
before_script:
- apk update && apk add curl openssh-client git grep xmlstarlet
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
- apk update && apk add brotli curl grep
script:
- sh src/script.sh
- find public -type f -regex '.*\.\(txt\|conf\|tpl\|rules\)$' -exec gzip -f -k -9 {} \;
- find public -type f -regex '.*\.\(txt\|conf\|tpl\|rules\)$' -exec brotli -f -k -9 {} \;
- git checkout master
- git config --global user.name "curben-bot"
- git config --global user.email "3048979-curben-bot@users.noreply.gitlab.com"
# Commit the changes
- sh src/commit.sh
# Generate successful status badge
- mkdir -p .gitlab/
- sh src/badge.sh "success"
- git add .gitlab/status.svg
# Only commit when diff exists https://stackoverflow.com/a/8123841
- git diff-index --quiet HEAD || git commit -m "Success pipeline"
- ssh -T git@gitlab.com
- git remote set-url origin git@gitlab.com:curben/phishing-filter.git
- git push origin master
rules:
# Only trigger through schedule job and "Run pipeline" in master branch
- if: '$CI_COMMIT_REF_NAME == "master" && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
when: always
# Upload working folder as a job artifact
artifacts:
paths:
- tmp/
expire_in: 30 days
- tmp
- public
deploy_job: # Trigger deploy job on curben/phishing-filter-mirror
pages:
stage: deploy
trigger:
project: curben/phishing-filter-mirror
branch: master
strategy: depend
rules:
- if: '$CI_COMMIT_REF_NAME == "master" && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
when: always
failed_job:
stage: failed_stage
before_script:
- 'which ssh-agent || (apk update && apk add curl openssh-client git grep)'
- eval $(ssh-agent -s)
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
- chmod 644 ~/.ssh/known_hosts
script:
- git checkout master
- git config --global user.name "curben-bot"
- git config --global user.email "3048979-curben-bot@users.noreply.gitlab.com"
- echo
- mkdir -p .gitlab/
- sh src/badge.sh "failed"
- git add .gitlab/status.svg
- git diff-index --quiet HEAD || git commit -m "Failed pipeline"
- ssh -T git@gitlab.com
- git remote set-url origin git@gitlab.com:curben/phishing-filter.git
- git push origin master
artifacts:
paths:
- public
rules:
- if: '$CI_COMMIT_REF_NAME == "master" && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
# Run this job only when deploy_job failed
when: on_failure
# Only trigger through schedule job and "Run pipeline" in main branch
- if: '$CI_COMMIT_REF_NAME == "main" && ($CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web")'
when: always
include:
- template: Security/Secret-Detection.gitlab-ci.yml

1
.gitlab/status.svg
View File

@ -1 +0,0 @@
<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="102" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="102" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h53v20H0z"/><path fill="#4c1" d="M53 0h49v20H53z"/><path fill="url(#b)" d="M0 0h102v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"> <text x="275" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">pipeline</text><text x="275" y="140" transform="scale(.1)" textLength="430">pipeline</text><text x="765" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="390">passed</text><text x="765" y="140" transform="scale(.1)" textLength="390">passed</text></g> </svg>
Side by Side

Before

Width:  |  Height:  |  Size: 959 B

1
.nvmrc Normal file
View File

@ -0,0 +1 @@
lts/*

119
README.md
View File

@ -1,5 +1,7 @@
# Phishing URL Blocklist
> Edit 2021/01/08: the default branch has changed to **main**.
A blocklist of phishing websites, based on the [PhishTank](https://www.phishtank.com/) and [OpenPhish](https://openphish.com/) lists. Blocklist is updated twice a day.
There are multiple formats available, refer to the appropriate section according to the program used:
@ -38,12 +40,7 @@ _included by default in uBO >=[1.39.0](https://github.com/gorhill/uBlock/release
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter.txt
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter.txt
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter.txt
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter.txt
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter.txt
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter.txt
- https://curben.gitlab.io/phishing-filter/phishing-filter.txt
</details>
@ -58,12 +55,7 @@ Import the following URL into AdGuard browser extension to subscribe:
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-ag.txt
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-ag.txt
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-ag.txt
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-ag.txt
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-ag.txt
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-ag.txt
- https://curben.gitlab.io/phishing-filter/phishing-filter-ag.txt
</details>
@ -78,12 +70,7 @@ Import the following URL into Vivaldi's **Tracker Blocking Sources** to subscrib
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-vivaldi.txt
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-vivaldi.txt
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-vivaldi.txt
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-vivaldi.txt
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-vivaldi.txt
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-vivaldi.txt
- https://curben.gitlab.io/phishing-filter/phishing-filter-vivaldi.txt
</details>
@ -96,12 +83,7 @@ This blocklist includes domains and IP addresses.
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-domains.txt
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-domains.txt
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-domains.txt
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-domains.txt
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-domains.txt
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-domains.txt
- https://curben.gitlab.io/phishing-filter/phishing-filter-domains.txt
</details>
@ -114,12 +96,7 @@ This AdGuard Home-compatible blocklist includes domains and IP addresses.
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-agh.txt
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-agh.txt
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-agh.txt
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-agh.txt
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-agh.txt
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/phishing-filter-agh.txt
- https://curben.gitlab.io/phishing-filter/phishing-filter-agh.txt
</details>
@ -132,12 +109,7 @@ This blocklist includes domains only.
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-hosts.txt
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-hosts.txt
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-hosts.txt
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-hosts.txt
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-hosts.txt
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-hosts.txt
- https://curben.gitlab.io/phishing-filter/phishing-filter-hosts.txt
</details>
@ -166,12 +138,7 @@ printf "\nconf-file=/usr/local/etc/dnsmasq/phishing-filter-dnsmasq.conf\n" >> /e
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-dnsmasq.conf
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-dnsmasq.conf
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-dnsmasq.conf
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-dnsmasq.conf
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-dnsmasq.conf
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-dnsmasq.conf
- https://curben.gitlab.io/phishing-filter/phishing-filter-dnsmasq.conf
</details>
@ -218,12 +185,7 @@ Zone file is derived from [here](https://github.com/tomzuu/blacklist-named/blob/
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-bind.conf
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-bind.conf
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-bind.conf
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-bind.conf
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-bind.conf
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-bind.conf
- https://curben.gitlab.io/phishing-filter/phishing-filter-bind.conf
</details>
@ -236,12 +198,7 @@ This blocklist includes domains only.
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-rpz.conf
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-rpz.conf
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-rpz.conf
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-rpz.conf
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-rpz.conf
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-rpz.conf
- https://curben.gitlab.io/phishing-filter/phishing-filter-rpz.conf
</details>
@ -270,12 +227,7 @@ printf '\n include: "/usr/local/etc/unbound/phishing-filter-unbound.conf"\n' >>
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-unbound.conf
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-unbound.conf
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-unbound.conf
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-unbound.conf
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-unbound.conf
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-unbound.conf
- https://curben.gitlab.io/phishing-filter/phishing-filter-unbound.conf
</details>
@ -311,19 +263,10 @@ Configure dnscrypt-proxy to use the blocklist:
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-dnscrypt-blocked-names.txt
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-dnscrypt-blocked-names.txt
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-dnscrypt-blocked-names.txt
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-dnscrypt-blocked-names.txt
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-dnscrypt-blocked-names.txt
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-dnscrypt-blocked-names.txt
- https://curben.gitlab.io/phishing-filter/phishing-filter-dnscrypt-blocked-names.txt
- https://curben.gitlab.io/phishing-filter/phishing-filter-dnscrypt-blocked-ips.txt
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-dnscrypt-blocked-ips.txt
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-dnscrypt-blocked-ips.txt
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-dnscrypt-blocked-ips.txt
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-dnscrypt-blocked-ips.txt
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-dnscrypt-blocked-ips.txt
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-dnscrypt-blocked-ips.txt
</details>
## Tracking Protection List (IE)
@ -335,12 +278,7 @@ This blocklist includes domains only.
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter.tpl
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter.tpl
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter.tpl
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter.tpl
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter.tpl
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter.tpl
- https://curben.gitlab.io/phishing-filter/phishing-filter.tpl
</details>
@ -369,12 +307,7 @@ printf "\ninclude \$RULE_PATH/phishing-filter-snort2.rules\n" >> /etc/snort/snor
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-snort2.rules
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-snort2.rules
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-snort2.rules
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-snort2.rules
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-snort2.rules
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-snort2.rules
- https://curben.gitlab.io/phishing-filter/phishing-filter-snort2.rules
</details>
@ -411,12 +344,7 @@ ips =
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-snort3.rules
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-snort3.rules
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-snort3.rules
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-snort3.rules
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-snort3.rules
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-snort3.rules
- https://curben.gitlab.io/phishing-filter/phishing-filter-snort3.rules
</details>
@ -451,12 +379,7 @@ rule-files:
<details>
<summary>Mirrors</summary>
- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/phishing-filter-suricata.rules
- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/phishing-filter-suricata.rules
- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/phishing-filter-suricata.rules
- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/phishing-filter-suricata.rules
- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/phishing-filter-suricata.rules
- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/phishing-filter-suricata.rules
- https://curben.gitlab.io/phishing-filter/phishing-filter-suricata.rules
</details>
@ -484,9 +407,7 @@ Use shallow clone to get the recent revisions only. Getting the last five revisi
[src/](src/): [CC0](LICENSE.md)
[dist/](dist/): [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)
[badge.sh](src/badge.sh) & [.gitlab/](.gitlab/) contain badges that are licensed by [Shields.io](https://shields.io) under [CC0 1.0](LICENSE.md)
filters: [CC BY-SA 4.0](https://creativecommons.org/licenses/by-sa/4.0/)
[PhishTank](https://www.phishtank.com/): Available [free of charge](https://www.phishtank.com/faq.php#isitoktousetheapifor) by Cisco for commercial and non-commercial use.

5998
dist/phishing-filter-ag.txt vendored
View File

File diff suppressed because it is too large Load Diff

4489
dist/phishing-filter-agh.txt vendored
View File

File diff suppressed because it is too large Load Diff

4455
dist/phishing-filter-bind.conf vendored
View File

File diff suppressed because it is too large Load Diff

41
dist/phishing-filter-dnscrypt-blocked-ips.txt vendored
View File

@ -1,41 +0,0 @@
# Title: Phishing IPs Blocklist
# Updated: Sat, 08 Jan 2022 00:01:30 +0000
# Expires: 1 day (update frequency)
# Homepage: https://gitlab.com/curben/phishing-filter
# License: https://gitlab.com/curben/phishing-filter#license
# Source: https://www.phishtank.com/ & https://openphish.com/
# Notice: https://curben.gitlab.io/phishing-filter-mirror is moved to https://curben.gitlab.io/malware-filter
101.32.192.174
103.114.16.4
104.168.173.244
104.168.173.248
107.172.198.119
113.164.17.147
119.28.91.122
124.156.136.189
130.211.30.154
14.98.234.77
141.193.196.74
149.210.143.165
161.35.142.2
161.35.56.215
165.227.122.125
179.48.65.130
182.73.136.210
2.136.95.251
20.206.88.15
208.82.115.230
211.57.201.45
216.244.165.236
35.192.38.184
35.199.84.117
42.193.110.254
45.186.132.130
45.9.20.146
47.74.89.4
52.148.252.166
78.108.89.240
8.209.107.30
8.210.12.187
85.202.169.200

4455
dist/phishing-filter-dnscrypt-blocked-names.txt vendored
View File

File diff suppressed because it is too large Load Diff

4455
dist/phishing-filter-dnsmasq.conf vendored
View File

File diff suppressed because it is too large Load Diff

4488
dist/phishing-filter-domains.txt vendored
View File

File diff suppressed because it is too large Load Diff

4455
dist/phishing-filter-hosts.txt vendored
View File

File diff suppressed because it is too large Load Diff

4460
dist/phishing-filter-rpz.conf vendored
View File

File diff suppressed because it is too large Load Diff

5997
dist/phishing-filter-snort2.rules vendored
View File

File diff suppressed because it is too large Load Diff

5997
dist/phishing-filter-snort3.rules vendored
View File

File diff suppressed because it is too large Load Diff

5997
dist/phishing-filter-suricata.rules vendored
View File

File diff suppressed because it is too large Load Diff

4455
dist/phishing-filter-unbound.conf vendored
View File

File diff suppressed because it is too large Load Diff

5998
dist/phishing-filter-vivaldi.txt vendored
View File

File diff suppressed because it is too large Load Diff

4458
dist/phishing-filter.tpl vendored
View File

File diff suppressed because it is too large Load Diff

5998
dist/phishing-filter.txt vendored
View File

File diff suppressed because it is too large Load Diff

14
package.json Normal file
View File

@ -0,0 +1,14 @@
{
"name": "phishing-filter",
"private": true,
"scripts": {
"build": "node src/build.js"
},
"dependencies": {
"extract-zip": "^2.0.1",
"got": "^11.8.3"
},
"engines": {
"node": ">= 14.15.0"
}
}

15
src/badge.sh
View File

@ -1,15 +0,0 @@
#!/bin/sh
set -efux -o pipefail
## Set status badge
successBadge='<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="102" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="102" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h53v20H0z"/><path fill="#4c1" d="M53 0h49v20H53z"/><path fill="url(#b)" d="M0 0h102v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"> <text x="275" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">pipeline</text><text x="275" y="140" transform="scale(.1)" textLength="430">pipeline</text><text x="765" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="390">passed</text><text x="765" y="140" transform="scale(.1)" textLength="390">passed</text></g> </svg>'
failedBadge='<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="92" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="92" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h53v20H0z"/><path fill="#e05d44" d="M53 0h39v20H53z"/><path fill="url(#b)" d="M0 0h92v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"> <text x="275" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">pipeline</text><text x="275" y="140" transform="scale(.1)" textLength="430">pipeline</text><text x="715" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="290">failed</text><text x="715" y="140" transform="scale(.1)" textLength="290">failed</text></g> </svg>'
if [ "$1" = "success" ]; then
echo "$successBadge" > ".gitlab/status.svg"
elif [ "$1" = "failed" ]; then
echo "$failedBadge" > ".gitlab/status.svg"
fi

30
src/build.js Normal file
View File

@ -0,0 +1,30 @@
'use strict'
// for deployment outside of GitLab CI, e.g. Cloudflare Pages and Netlify
const { stream: gotStream } = require('got')
const unzip = require('extract-zip')
const { join } = require('path')
const { mkdir } = require('fs/promises')
const { createWriteStream } = require('fs')
const { pipeline } = require('stream/promises')
const rootPath = join(__dirname, '..')
const tmpPath = join(rootPath, 'tmp')
const zipPath = join(tmpPath, 'artifacts.zip')
const artifactsUrl = 'https://gitlab.com/curben/phishing-filter/-/jobs/artifacts/main/download?job=pages'
const f = async () => {
await mkdir(tmpPath, { recursive: true })
console.log(`Downloading artifacts.zip from "${artifactsUrl}"`)
await pipeline(
gotStream(artifactsUrl),
createWriteStream(zipPath)
)
console.log('Extracting artifacts.zip...')
await unzip(zipPath, { dir: rootPath })
}
f()

11
src/commit.sh
View File

@ -1,11 +0,0 @@
#!/bin/sh
set -efux -o pipefail
## Commit the filter update
## GitLab CI does not permit shell variable in .gitlab-ci.yml.
## This file is a workaround for that.
CURRENT_TIME="$(date -R -u)"
git commit -a -m "Filter updated: $CURRENT_TIME"

62
src/script.sh
View File

@ -150,13 +150,15 @@ THIRD_LINE="! Expires: 1 day (update frequency)"
FOURTH_LINE="! Homepage: https://gitlab.com/curben/phishing-filter"
FIFTH_LINE="! License: https://gitlab.com/curben/phishing-filter#license"
SIXTH_LINE="! Source: https://www.phishtank.com/ & https://openphish.com/"
NOTICE="\n! Notice: https://curben.gitlab.io/phishing-filter-mirror is moved to https://curben.gitlab.io/malware-filter\n"
COMMENT_UBO="$FIRST_LINE\n$SECOND_LINE\n$THIRD_LINE\n$FOURTH_LINE\n$FIFTH_LINE\n$SIXTH_LINE\n$NOTICE"
ANNOUNCEMENT_1="\n! 2021/01/08: There has been a major change to the mirrors, check the repo for the new mirrors."
ANNOUNCEMENT_2="! Old mirrors will be deprecated in 3 months. The main download link \"curben.gitlab.io/malware-filter/\" _is not affected_."
COMMENT_UBO="$FIRST_LINE\n$SECOND_LINE\n$THIRD_LINE\n$FOURTH_LINE\n$FIFTH_LINE\n$SIXTH_LINE\n$ANNOUNCEMENT_1\n$ANNOUNCEMENT_2"
mkdir -p "../public/"
cat "phishing-notop-domains.txt" "phishing-url-top-domains.txt" | \
sort | \
sed '1 i\'"$COMMENT_UBO"'' > "../dist/phishing-filter.txt"
sed '1 i\'"$COMMENT_UBO"'' > "../public/phishing-filter.txt"
# Adguard Home
@ -167,7 +169,7 @@ sed "s/$/^/g" > "phishing-domains-adguard-home.txt"
cat "phishing-domains-adguard-home.txt" | \
sort | \
sed '1 i\'"$COMMENT_UBO"'' | \
sed "1s/Blocklist/Blocklist (AdGuard Home)/" > "../dist/phishing-filter-agh.txt"
sed "1s/Blocklist/Blocklist (AdGuard Home)/" > "../public/phishing-filter-agh.txt"
# Adguard browser extension
@ -178,7 +180,7 @@ sed "s/$/\$all/g" > "phishing-domains-adguard.txt"
cat "phishing-domains-adguard.txt" "phishing-url-top-domains.txt" | \
sort | \
sed '1 i\'"$COMMENT_UBO"'' | \
sed "1s/Blocklist/Blocklist (AdGuard)/" > "../dist/phishing-filter-ag.txt"
sed "1s/Blocklist/Blocklist (AdGuard)/" > "../public/phishing-filter-ag.txt"
# Vivaldi
@ -190,7 +192,7 @@ cat "phishing-domains-vivaldi.txt" "phishing-url-top-domains.txt" | \
sed "s/\$all$/\$document/g" | \
sort | \
sed '1 i\'"$COMMENT_UBO"'' | \
sed "1s/Blocklist/Blocklist (Vivaldi)/" > "../dist/phishing-filter-vivaldi.txt"
sed "1s/Blocklist/Blocklist (Vivaldi)/" > "../public/phishing-filter-vivaldi.txt"
## Domains-only blocklist
@ -199,7 +201,7 @@ COMMENT=$(printf "$COMMENT_UBO" | sed "s/^!/#/g" | sed "1s/URL/Domains/" | awk '
cat "phishing-notop-domains.txt" | \
sort | \
sed '1 i\'"$COMMENT"'' > "../dist/phishing-filter-domains.txt"
sed '1 i\'"$COMMENT"'' > "../public/phishing-filter-domains.txt"
cat "phishing-notop-domains.txt" | \
grep -vE "^([0-9]{1,3}[\.]){3}[0-9]{1,3}$" > "phishing-notop-hosts.txt"
@ -209,7 +211,7 @@ cat "phishing-notop-hosts.txt" | \
sed "s/^/0.0.0.0 /g" | \
# Re-insert comment
sed '1 i\'"$COMMENT"'' | \
sed "1s/Domains/Hosts/" > "../dist/phishing-filter-hosts.txt"
sed "1s/Domains/Hosts/" > "../public/phishing-filter-hosts.txt"
## Dnsmasq-compatible blocklist
@ -217,7 +219,7 @@ cat "phishing-notop-hosts.txt" | \
sed "s/^/address=\//g" | \
sed "s/$/\/0.0.0.0/g" | \
sed '1 i\'"$COMMENT"'' | \
sed "1s/Blocklist/dnsmasq Blocklist/" > "../dist/phishing-filter-dnsmasq.conf"
sed "1s/Blocklist/dnsmasq Blocklist/" > "../public/phishing-filter-dnsmasq.conf"
## BIND-compatible blocklist
@ -225,7 +227,7 @@ cat "phishing-notop-hosts.txt" | \
sed 's/^/zone "/g' | \
sed 's/$/" { type master; notify no; file "null.zone.file"; };/g' | \
sed '1 i\'"$COMMENT"'' | \
sed "1s/Blocklist/BIND Blocklist/" > "../dist/phishing-filter-bind.conf"
sed "1s/Blocklist/BIND Blocklist/" > "../public/phishing-filter-bind.conf"
## DNS Response Policy Zone (RPZ)
@ -237,7 +239,7 @@ sed "s/$/ CNAME ./g" | \
sed '1 i\'"$RPZ_SYNTAX"'' | \
sed '1 i\'"$COMMENT"'' | \
sed "s/^#/;/g" | \
sed "1s/Blocklist/RPZ Blocklist/" > "../dist/phishing-filter-rpz.conf"
sed "1s/Blocklist/RPZ Blocklist/" > "../public/phishing-filter-rpz.conf"
## Unbound-compatible blocklist
@ -245,29 +247,29 @@ cat "phishing-notop-hosts.txt" | \
sed 's/^/local-zone: "/g' | \
sed 's/$/" always_nxdomain/g' | \
sed '1 i\'"$COMMENT"'' | \
sed "1s/Blocklist/Unbound Blocklist/" > "../dist/phishing-filter-unbound.conf"
sed "1s/Blocklist/Unbound Blocklist/" > "../public/phishing-filter-unbound.conf"
## dnscrypt-proxy blocklists
# name-based
cat "phishing-notop-hosts.txt" | \
sed '1 i\'"$COMMENT"'' | \
sed "1s/Domains/Names/" > "../dist/phishing-filter-dnscrypt-blocked-names.txt"
sed "1s/Domains/Names/" > "../public/phishing-filter-dnscrypt-blocked-names.txt"
# IPv4-based
cat "phishing-notop-domains.txt" | \
sort | \
grep -E "^([0-9]{1,3}[\.]){3}[0-9]{1,3}$" | \
sed '1 i\'"$COMMENT"'' | \
sed "1s/Domains/IPs/" > "../dist/phishing-filter-dnscrypt-blocked-ips.txt"
sed "1s/Domains/IPs/" > "../public/phishing-filter-dnscrypt-blocked-ips.txt"
set +x
## Snort & Suricata rulesets
rm -f "../dist/phishing-filter-snort2.rules" \
"../dist/phishing-filter-snort3.rules" \
"../dist/phishing-filter-suricata.rules"
rm -f "../public/phishing-filter-snort2.rules" \
"../public/phishing-filter-snort3.rules" \
"../public/phishing-filter-suricata.rules"
SID="200000001"
while read DOMAIN; do
@ -277,9 +279,9 @@ while read DOMAIN; do
SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"phishing-filter phishing website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.host; content:\"$DOMAIN\"; classtype:attempted-recon; sid:$SID; rev:1;)"
echo "$SN_RULE" >> "../dist/phishing-filter-snort2.rules"
echo "$SN3_RULE" >> "../dist/phishing-filter-snort3.rules"
echo "$SR_RULE" >> "../dist/phishing-filter-suricata.rules"
echo "$SN_RULE" >> "../public/phishing-filter-snort2.rules"
echo "$SN3_RULE" >> "../public/phishing-filter-snort3.rules"
echo "$SR_RULE" >> "../public/phishing-filter-suricata.rules"
SID=$(( $SID + 1 ))
done < "phishing-notop-domains.txt"
@ -295,23 +297,23 @@ while read URL; do
SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"phishing-filter phishing website detected\"; flow:established,from_client; http.method; content:\"GET\"; http.uri; content:\"$URI\"; endswith; nocase; http.host; content:\"$HOST\"; classtype:attempted-recon; sid:$SID; rev:1;)"
echo "$SN_RULE" >> "../dist/phishing-filter-snort2.rules"
echo "$SN3_RULE" >> "../dist/phishing-filter-snort3.rules"
echo "$SR_RULE" >> "../dist/phishing-filter-suricata.rules"
echo "$SN_RULE" >> "../public/phishing-filter-snort2.rules"
echo "$SN3_RULE" >> "../public/phishing-filter-snort3.rules"
echo "$SR_RULE" >> "../public/phishing-filter-suricata.rules"
SID=$(( $SID + 1 ))
done < "phishing-url-top-domains-raw.txt"
set -x
sed -i '1 i\'"$COMMENT"'' "../dist/phishing-filter-snort2.rules"
sed -i "1s/Domains Blocklist/URL Snort2 Ruleset/" "../dist/phishing-filter-snort2.rules"
sed -i '1 i\'"$COMMENT"'' "../public/phishing-filter-snort2.rules"
sed -i "1s/Domains Blocklist/URL Snort2 Ruleset/" "../public/phishing-filter-snort2.rules"
sed -i '1 i\'"$COMMENT"'' "../dist/phishing-filter-snort3.rules"
sed -i "1s/Domains Blocklist/URL Snort3 Ruleset/" "../dist/phishing-filter-snort3.rules"
sed -i '1 i\'"$COMMENT"'' "../public/phishing-filter-snort3.rules"
sed -i "1s/Domains Blocklist/URL Snort3 Ruleset/" "../public/phishing-filter-snort3.rules"
sed -i '1 i\'"$COMMENT"'' "../dist/phishing-filter-suricata.rules"
sed -i "1s/Domains Blocklist/URL Suricata Ruleset/" "../dist/phishing-filter-suricata.rules"
sed -i '1 i\'"$COMMENT"'' "../public/phishing-filter-suricata.rules"
sed -i "1s/Domains Blocklist/URL Suricata Ruleset/" "../public/phishing-filter-suricata.rules"
## IE blocklist
@ -320,7 +322,7 @@ COMMENT_IE="msFilterList\n$COMMENT\n: Expires=1\n#"
cat "phishing-notop-hosts.txt" | \
sed "s/^/-d /g" | \
sed '1 i\'"$COMMENT_IE"'' | \
sed "2s/Domains Blocklist/Hosts Blocklist (IE)/" > "../dist/phishing-filter.tpl"
sed "2s/Domains Blocklist/Hosts Blocklist (IE)/" > "../public/phishing-filter.tpl"
## Clean up artifacts