refactor: deploy filters to gitlab pages

- 8c94ddba40

.gitignore

@@ -1 +1,4 @@
 tmp/
+.vscode/
+node_modules/
+public/

.gitlab-ci.yml

@@ -1,87 +1,33 @@
-stages:
-  - test
-  - failed_stage
+image: alpine:latest
 
-image: node:lts-alpine # Use latest LTS version of Node.js on Alpine
-
-deploy_job:
-  stage: test
+build_job:
+  stage: build
 
   before_script:
-    - 'which ssh-agent || (apk update && apk add git openssh-client)'
-    - eval $(ssh-agent -s)
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
-    - chmod 644 ~/.ssh/known_hosts
+    - apk update && apk add brotli curl grep
 
   script:
     - sh src/script.sh
+    - find public -type f -regex '.*\.\(txt\|conf\|tpl\|rules\)$' -exec gzip -f -k -9 {} \;
+    - find public -type f -regex '.*\.\(txt\|conf\|tpl\|rules\)$' -exec brotli -f -k -9 {} \;
 
-    - git checkout master
-    - git config --global user.name "curben-bot"
-    - git config --global user.email "3048979-curben-bot@users.noreply.gitlab.com"
-
-    # Commit the changes
-    - sh src/commit.sh
-
-    # Generate successful status badge
-    - mkdir -p .gitlab/
-    - sh src/badge.sh "success"
-    - git add .gitlab/status.svg
-    # Only commit when diff exists https://stackoverflow.com/a/8123841
-    - git diff-index --quiet HEAD || git commit -m "Success pipeline"
-
-    - ssh -T git@gitlab.com
-    - git remote set-url origin git@gitlab.com:curben/tracking-filter.git
-    - git push origin master
-
-  rules:
-    # Only trigger through schedule job in master branch
-    - if: '$CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule"'
-      when: always
-    # Only trigger through "Run pipeline" in master branch
-    - if: '$CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "web"'
-      when: always
-
-  # Upload working folder as a job artifact
   artifacts:
     paths:
-      - tmp/
-    expire_in: 30 days
+      - tmp
+      - public
 
-failed_job:
-  stage: failed_stage
-
-  before_script:
-    - 'which ssh-agent || (apk update && apk add openssh-client git)'
-    - eval $(ssh-agent -s)
-    - echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add - > /dev/null
-    - mkdir -p ~/.ssh
-    - chmod 700 ~/.ssh
-    - echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
-    - chmod 644 ~/.ssh/known_hosts
+pages:
+  stage: deploy
 
   script:
-    - git checkout master
-    - git config --global user.name "curben-bot"
-    - git config --global user.email "3048979-curben-bot@users.noreply.gitlab.com"
+    - echo
 
-    - mkdir -p .gitlab/
-    - sh src/badge.sh "failed"
-    - git add .gitlab/status.svg
-    - git diff-index --quiet HEAD || git commit -m "Failed pipeline"
-    - ssh -T git@gitlab.com
-    - git remote set-url origin git@gitlab.com:curben/tracking-filter.git
-    - git push origin master
+  artifacts:
+    paths:
+      - public
 
   rules:
-    - if: '$CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule"'
-      # Run this job only when deploy_job failed
-      when: on_failure
-    - if: '$CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "web"'
-      when: on_failure
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
 
 include:
   - template: Security/Secret-Detection.gitlab-ci.yml

.gitlab/status.svg

@@ -1 +0,0 @@
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="102" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="102" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h53v20H0z"/><path fill="#4c1" d="M53 0h49v20H53z"/><path fill="url(#b)" d="M0 0h102v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"> <text x="275" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">pipeline</text><text x="275" y="140" transform="scale(.1)" textLength="430">pipeline</text><text x="765" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="390">passed</text><text x="765" y="140" transform="scale(.1)" textLength="390">passed</text></g> </svg>

README.md

@@ -1,5 +1,7 @@
 # Tracking JS Blocklist
 
+> Edit 2021/01/08: the default branch has changed to **main**.
+
 A blocklist of javascript links (or webpages with javascript) that perform [browser fingerprinting](https://en.wikipedia.org/wiki/Web_tracking). The original data set is collected by DuckDuckGo's [Tracker Radar](https://github.com/duckduckgo/tracker-radar) project. Blocklist is updated on the first day of the month.
 
 The list contains **all** the links that Tracker Radar detected to have browser fingerprinting, there is no further processing to remove false positive.
@@ -23,12 +25,7 @@ Import the following URL into uBO/AdGuard to subscribe:
 <details>
 <summary>Mirrors</summary>
 
-- https://cdn.statically.io/gl/curben/phishing-filter/master/dist/tracking-filter.txt
-- https://glcdn.githack.com/curben/phishing-filter/raw/master/dist/tracking-filter.txt
-- https://raw.githubusercontent.com/curbengh/phishing-filter/master/dist/tracking-filter.txt
-- https://cdn.statically.io/gh/curbengh/phishing-filter/master/dist/tracking-filter.txt
-- https://gitcdn.xyz/repo/curbengh/phishing-filter/master/dist/tracking-filter.txt
-- https://cdn.jsdelivr.net/gh/curbengh/phishing-filter/dist/tracking-filter.txt
+- https://curben.gitlab.io/tracking-filter/tracking-filter.txt
 
 </details>
 
@@ -43,12 +40,7 @@ Import the following URL into Vivaldi's **Tracker Blocking Sources** to subscrib
 <details>
 <summary>Mirrors</summary>
 
-- https://cdn.statically.io/gl/curben/tracking-filter/master/dist/tracking-filter-vivaldi.txt
-- https://glcdn.githack.com/curben/tracking-filter/raw/master/dist/tracking-filter-vivaldi.txt
-- https://raw.githubusercontent.com/curbengh/tracking-filter/master/dist/tracking-filter-vivaldi.txt
-- https://cdn.statically.io/gh/curbengh/tracking-filter/master/dist/tracking-filter-vivaldi.txt
-- https://gitcdn.xyz/repo/curbengh/tracking-filter/master/dist/tracking-filter-vivaldi.txt
-- https://cdn.jsdelivr.net/gh/curbengh/tracking-filter/dist/tracking-filter-vivaldi.txt
+- https://curben.gitlab.io/tracking-filter/tracking-filter-vivaldi.txt
 
 </details>
 
@@ -61,12 +53,7 @@ A list of URLs without any filter syntax.
 <details>
 <summary>Mirrors</summary>
 
-- https://cdn.statically.io/gl/curben/tracking-filter/master/dist/tracking-data.txt
-- https://glcdn.githack.com/curben/tracking-filter/raw/master/dist/tracking-data.txt
-- https://raw.githubusercontent.com/curbengh/tracking-filter/master/dist/tracking-data.txt
-- https://cdn.statically.io/gh/curbengh/tracking-filter/master/dist/tracking-data.txt
-- https://gitcdn.xyz/repo/curbengh/tracking-filter/master/dist/tracking-data.txt
-- https://cdn.jsdelivr.net/gh/curbengh/tracking-filter/dist/tracking-data.txt
+- https://curben.gitlab.io/tracking-filter/tracking-data.txt
 
 </details>
 
@@ -78,9 +65,7 @@ A list of URLs without any filter syntax.
 
 [src/](src/): [CC0 1.0](LICENSE.md)
 
-[badge.sh](src/badge.sh) & [.gitlab/](.gitlab/) contain badges that are licensed by [Shields.io](https://shields.io) under [CC0 1.0](LICENSE.md)
-
-[dist/](dist/): [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/)
+filters: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/)
 
 DuckDuckGo Tracker Radar data: [CC BY-NC-SA 4.0](https://creativecommons.org/licenses/by-nc-sa/4.0/)
 

package.json

@@ -0,0 +1,14 @@
+{
+  "name": "tracking-filter",
+  "private": true,
+  "scripts": {
+    "build": "node src/build.js"
+  },
+  "dependencies": {
+    "extract-zip": "^2.0.1",
+    "got": "^11.8.3"
+  },
+  "engines": {
+    "node": ">= 14.15.0"
+  }
+}

src/badge.sh

@@ -1,15 +0,0 @@
-#!/bin/sh
-
-set -efux -o pipefail
-
-## Set status badge
-
-successBadge='<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="102" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="102" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h53v20H0z"/><path fill="#4c1" d="M53 0h49v20H53z"/><path fill="url(#b)" d="M0 0h102v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"> <text x="275" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">pipeline</text><text x="275" y="140" transform="scale(.1)" textLength="430">pipeline</text><text x="765" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="390">passed</text><text x="765" y="140" transform="scale(.1)" textLength="390">passed</text></g> </svg>'
-
-failedBadge='<svg xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" width="92" height="20"><linearGradient id="b" x2="0" y2="100%"><stop offset="0" stop-color="#bbb" stop-opacity=".1"/><stop offset="1" stop-opacity=".1"/></linearGradient><clipPath id="a"><rect width="92" height="20" rx="3" fill="#fff"/></clipPath><g clip-path="url(#a)"><path fill="#555" d="M0 0h53v20H0z"/><path fill="#e05d44" d="M53 0h39v20H53z"/><path fill="url(#b)" d="M0 0h92v20H0z"/></g><g fill="#fff" text-anchor="middle" font-family="DejaVu Sans,Verdana,Geneva,sans-serif" font-size="110"> <text x="275" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="430">pipeline</text><text x="275" y="140" transform="scale(.1)" textLength="430">pipeline</text><text x="715" y="150" fill="#010101" fill-opacity=".3" transform="scale(.1)" textLength="290">failed</text><text x="715" y="140" transform="scale(.1)" textLength="290">failed</text></g> </svg>'
-
-if [ "$1" = "success" ]; then
-  echo "$successBadge" > ".gitlab/status.svg"
-elif [ "$1" = "failed" ]; then
-  echo "$failedBadge" > ".gitlab/status.svg"
-fi

src/build.js

@@ -0,0 +1,30 @@
+'use strict'
+
+// for deployment outside of GitLab CI, e.g. Cloudflare Pages and Netlify
+
+const { stream: gotStream } = require('got')
+const unzip = require('extract-zip')
+const { join } = require('path')
+const { mkdir } = require('fs/promises')
+const { createWriteStream } = require('fs')
+const { pipeline } = require('stream/promises')
+
+const rootPath = join(__dirname, '..')
+const tmpPath = join(rootPath, 'tmp')
+const zipPath = join(tmpPath, 'artifacts.zip')
+const artifactsUrl = 'https://gitlab.com/curben/tracking-filter/-/jobs/artifacts/main/download?job=pages'
+
+const f = async () => {
+  await mkdir(tmpPath, { recursive: true })
+
+  console.log(`Downloading artifacts.zip from "${artifactsUrl}"`)
+  await pipeline(
+    gotStream(artifactsUrl),
+    createWriteStream(zipPath)
+  )
+
+  console.log('Extracting artifacts.zip...')
+  await unzip(zipPath, { dir: rootPath })
+}
+
+f()

src/commit.sh

@@ -1,11 +0,0 @@
-#!/bin/sh
-
-set -efux -o pipefail
-
-## Commit the update
-
-## GitLab CI does not permit shell variable in .gitlab-ci.yml.
-## This file is a workaround for that.
-
-CURRENT_TIME="$(date -R -u)"
-git commit -a -m "Updated: $CURRENT_TIME"

src/script.js

@@ -1,7 +1,7 @@
 'use strict'
 
 const { join } = require('path')
-const { appendFile, readdir, readFile, rm } = require('fs').promises
+const { appendFile, readdir, readFile, rm } = require('fs/promises')
 const { parse } = JSON
 
 const f = async () => {

src/script.sh

@@ -27,25 +27,29 @@ THIRD_LINE="! Expires: 7 day (update frequency)"
 FOURTH_LINE="! Homepage: https://gitlab.com/curben/tracking-filter"
 FIFTH_LINE="! License: https://gitlab.com/curben/tracking-filter#license"
 SIXTH_LINE="! Source: https://github.com/duckduckgo/tracker-radar"
-COMMENT_UBO="$FIRST_LINE\n$SECOND_LINE\n$THIRD_LINE\n$FOURTH_LINE\n$FIFTH_LINE\n$SIXTH_LINE"
+ANNOUNCEMENT_1="\n! 2021/01/08: There has been a major change to the mirrors, check the repo for the new mirrors."
+ANNOUNCEMENT_2="! Old mirrors will be deprecated in 3 months. The main download link \"curben.gitlab.io/malware-filter/\" _is not affected_."
+COMMENT_UBO="$FIRST_LINE\n$SECOND_LINE\n$THIRD_LINE\n$FOURTH_LINE\n$FIFTH_LINE\n$SIXTH_LINE\n$ANNOUNCEMENT_1\n$ANNOUNCEMENT_2"
+
+mkdir -p "../public/"
 
 # Original data
 cat "tracking-url.txt" | \
 sed '1 i\'"$COMMENT_UBO"'' | \
-sed "s/^!/#/g" > "../dist/tracking-data.txt"
+sed "s/^!/#/g" > "../public/tracking-data.txt"
 
 # uBO & Adguard
 cat "tracking-url.txt" | \
 sed "s/^/||/g" | \
 sed "s/$/\$all/g" | \
-sed '1 i\'"$COMMENT_UBO"'' > "../dist/tracking-filter.txt"
+sed '1 i\'"$COMMENT_UBO"'' > "../public/tracking-filter.txt"
 
 # Vivaldi
 cat "tracking-url.txt" | \
 sed "s/^/||/g" | \
 sed "s/$/\$document/g" | \
 sed '1 i\'"$COMMENT_UBO"'' | \
-sed "1s/Blocklist/Blocklist (Vivaldi)/" > "../dist/tracking-filter-vivaldi.txt"
+sed "1s/Blocklist/Blocklist (Vivaldi)/" > "../public/tracking-filter-vivaldi.txt"
 
 ## This took more than 10 minutes on my not-so-sham laptop which is equivalent of *forever* on puny CI/CD VM.
 # # Snort & Suricata
@@ -54,9 +58,9 @@ sed "1s/Blocklist/Blocklist (Vivaldi)/" > "../dist/tracking-filter-vivaldi.txt"
 # ## Temporarily disable command print
 # set +x
 
-# rm -f "../dist/tracking-filter-snort2.rules" \
-#   "../dist/tracking-filter-snort3.rules" \
-#   "../dist/tracking-filter-suricata.rules"
+# rm -f "../public/tracking-filter-snort2.rules" \
+#   "../public/tracking-filter-snort3.rules" \
+#   "../public/tracking-filter-suricata.rules"
 
 # SID="400000001"
 
@@ -71,9 +75,9 @@ sed "1s/Blocklist/Blocklist (Vivaldi)/" > "../dist/tracking-filter-vivaldi.txt"
 
 #   SR_RULE="alert http \$HOME_NET any -> \$EXTERNAL_NET any (msg:\"tracking-filter tracking link detected\"; flow:established,from_client; http.method; content:\"GET\"; http.uri; content:\"$URI\"; endswith; nocase; http.host; content:\"$HOST\"; classtype:attempted-recon; sid:$SID; rev:1;)"
 
-#   echo "$SN_RULE" >> "../dist/tracking-filter-snort2.rules"
-#   echo "$SN3_RULE" >> "../dist/tracking-filter-snort3.rules"
-#   echo "$SR_RULE" >> "../dist/tracking-filter-suricata.rules"
+#   echo "$SN_RULE" >> "../public/tracking-filter-snort2.rules"
+#   echo "$SN3_RULE" >> "../public/tracking-filter-snort3.rules"
+#   echo "$SR_RULE" >> "../public/tracking-filter-suricata.rules"
 
 #   SID=$(( $SID + 1 ))
 # done < "tracking-url.txt"
@@ -81,14 +85,14 @@ sed "1s/Blocklist/Blocklist (Vivaldi)/" > "../dist/tracking-filter-vivaldi.txt"
 # ## Re-enable command print
 # set -x
 
-# sed -i '1 i\'"$COMMENT_HASH"'' "../dist/tracking-filter-snort2.rules"
-# sed -i "1s/Blocklist/Snort2 Ruleset/" "../dist/tracking-filter-snort2.rules"
+# sed -i '1 i\'"$COMMENT_HASH"'' "../public/tracking-filter-snort2.rules"
+# sed -i "1s/Blocklist/Snort2 Ruleset/" "../public/tracking-filter-snort2.rules"
 
-# sed -i '1 i\'"$COMMENT_HASH"'' "../dist/tracking-filter-snort3.rules"
-# sed -i "1s/Blocklist/Snort3 Ruleset/" "../dist/tracking-filter-snort3.rules"
+# sed -i '1 i\'"$COMMENT_HASH"'' "../public/tracking-filter-snort3.rules"
+# sed -i "1s/Blocklist/Snort3 Ruleset/" "../public/tracking-filter-snort3.rules"
 
-# sed -i '1 i\'"$COMMENT_HASH"'' "../dist/tracking-filter-suricata.rules"
-# sed -i "1s/Blocklist/Suricata Ruleset/" "../dist/tracking-filter-suricata.rules"
+# sed -i '1 i\'"$COMMENT_HASH"'' "../public/tracking-filter-suricata.rules"
+# sed -i "1s/Blocklist/Suricata Ruleset/" "../public/tracking-filter-suricata.rules"
 
 
 ## Clean-up artifacts