fix: download filters from artifacts

- 8c94ddba40

.gitignore

@@ -0,0 +1,4 @@
+tmp/
+public/
+node_modules/
+.vscode/

.gitlab-ci.yml

@@ -1,43 +1,30 @@
 image: alpine:latest
 
-pages:
+build_job:
+  stage:
+    - build
+
   before_script:
-    - apk update && apk add git brotli curl
-    - git clone --depth 5 https://gitlab.com/curben/urlhaus-filter.git urlhaus-filter/
-    - git clone --depth 5 https://gitlab.com/curben/phishing-filter.git phishing-filter/
-    - git clone --depth 5 https://gitlab.com/curben/pup-filter.git pup-filter/
-
-    - mkdir -p oisd/
-    - curl -L https://abp.oisd.nl/basic/ -o oisd/oisd_abp_light.txt
-    - curl -L https://abp.oisd.nl/ -o oisd/oisd_abp.txt
-    - curl -L https://dbl.oisd.nl/basic/ -o oisd/oisd_dbl_light.txt
-    - curl -L https://dbl.oisd.nl/ -o oisd/oisd_dbl.txt
-    - curl -L https://dblw.oisd.nl/basic/ -o oisd/oisd_dblw_light.txt
-    - curl -L https://dblw.oisd.nl/ -o oisd/oisd_dblw.txt
-    - curl -L https://hosts.oisd.nl/basic/ -o oisd/oisd_hosts_light.txt
-    - curl -L https://hosts.oisd.nl/ -o oisd/oisd_hosts.txt
-    - curl -L https://dnsmasq.oisd.nl/basic/ -o oisd/oisd_dnsmasq_light.txt
-    - curl -L https://dnsmasq.oisd.nl/ -o oisd/oisd_dnsmasq.txt
-    - curl -L https://rpz.oisd.nl/basic/ -o oisd/oisd_rpz_light.txt
-    - curl -L https://rpz.oisd.nl/ -o oisd/oisd_rpz.txt
-
-    - mkdir -p tracking-filter/
-    - git clone --depth 5 https://gitlab.com/curben/tracking-filter.git tracking-filter/
+    - apk update && apk add brotli curl
 
   script:
-    - mkdir -p public/
-    - cp urlhaus-filter/urlhaus-filter* public/
-    - cp phishing-filter/dist/phishing-filter* public/
-    - cp pup-filter/dist/pup-filter* public/
-    - cp oisd/* public/
-    - cp tracking-filter/dist/* public/
-    - find public -type f -regex '.*\.\(txt\|conf\|tpl\|rules\)$' -exec gzip -f -k -9 {} \;
-    - find public -type f -regex '.*\.\(txt\|conf\|tpl\|rules\)$' -exec brotli -f -k -9 {} \;
+    - sh src/script.sh
 
   artifacts:
     paths:
-      - public/
+      - tmp
+      - public
+
+pages:
+  stage:
+    - deploy
+
+  script:
+    - echo
+
+  artifacts:
+    paths:
+      - public
 
   rules:
-    - if: '$CI_COMMIT_REF_NAME == "master" && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "schedule")'
-      when: always
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH

package.json

@@ -0,0 +1,14 @@
+{
+  "name": "malware-filter",
+  "private": true,
+  "scripts": {
+    "build": "node src/build.js"
+  },
+  "dependencies": {
+    "extract-zip": "^2.0.1",
+    "got": "^11.8.3"
+  },
+  "engines": {
+    "node": ">= 14.15.0"
+  }
+}

src/build.js

@@ -0,0 +1,30 @@
+'use strict'
+
+// for deployment outside of GitLab CI, e.g. Cloudflare Pages and Netlify
+
+const { stream: gotStream } = require('got')
+const unzip = require('extract-zip')
+const { join } = require('path')
+const { mkdir } = require('fs/promises')
+const { createWriteStream } = require('fs')
+const { pipeline } = require('stream/promises')
+
+const rootPath = join(__dirname, '..')
+const tmpPath = join(rootPath, 'tmp')
+const zipPath = join(tmpPath, 'artifacts.zip')
+const artifactsUrl = 'https://gitlab.com/curben/malware-filter/-/jobs/artifacts/main/download?job=pages'
+
+const f = async () => {
+  await mkdir(tmpPath, { recursive: true })
+
+  console.log(`Downloading artifacts.zip from "${artifactsUrl}"`)
+  await pipeline(
+    gotStream(artifactsUrl),
+    createWriteStream(zipPath)
+  )
+
+  console.log('Extracting artifacts.zip...')
+  await unzip(zipPath, { dir: rootPath })
+}
+
+f()

src/script.sh

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+set -efux -o pipefail
+
+alias cp="cp -f"
+alias curl="curl -L"
+alias mkdir="mkdir -p"
+alias unzip="unzip -jo"
+
+mkdir "tmp/"
+cd "tmp/"
+
+curl "https://gitlab.com/curben/urlhaus-filter/-/jobs/artifacts/main/download?job=pages" -o "urlhaus-filter.zip"
+curl "https://gitlab.com/curben/phishing-filter/-/jobs/artifacts/main/download?job=pages" -o "phishing-filter.zip"
+curl "https://gitlab.com/curben/pup-filter/-/jobs/artifacts/main/download?job=pages" -o "pup-filter.zip"
+curl "https://gitlab.com/curben/tracking-filter/-/jobs/artifacts/main/download?job=pages" -o "tracking-filter.zip"
+
+for zipfile in $(find . -name "*.zip" -type f)
+do
+  unzip "$zipfile"
+done
+
+curl "https://abp.oisd.nl/basic/" -o "oisd_abp_light.txt"
+curl "https://abp.oisd.nl/" -o "oisd_abp.txt"
+curl "https://dbl.oisd.nl/basic/" -o "oisd_dbl_light.txt"
+curl "https://dbl.oisd.nl/" -o "oisd_dbl.txt"
+curl "https://dblw.oisd.nl/basic/" -o "oisd_dblw_light.txt"
+curl "https://dblw.oisd.nl/" -o "oisd_dblw.txt"
+curl "https://hosts.oisd.nl/basic/" -o "oisd_hosts_light.txt"
+curl "https://hosts.oisd.nl/" -o "oisd_hosts.txt"
+curl "https://dnsmasq.oisd.nl/basic/" -o "oisd_dnsmasq_light.txt"
+curl "https://dnsmasq.oisd.nl/" -o "oisd_dnsmasq.txt"
+curl "https://rpz.oisd.nl/basic/" -o "oisd_rpz_light.txt"
+curl "https://rpz.oisd.nl/" -o "oisd_rpz.txt"
+
+find . -name "oisd*.txt" -type f -print0 | xargs -0 gzip -f -k -9
+find . -name "oisd*.txt" -type f -print0 | xargs -0 brotli -f -k -9
+
+mkdir "../public/"
+find . ! -name "*.zip" -type f | xargs cp -t "../public/"