diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..71c4a21 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +tmp/ +public/ +node_modules/ +.vscode/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 19b8f6a..de19e22 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,43 +1,30 @@ image: alpine:latest -pages: +build_job: + stage: + - build + before_script: - - apk update && apk add git brotli curl - - git clone --depth 5 https://gitlab.com/curben/urlhaus-filter.git urlhaus-filter/ - - git clone --depth 5 https://gitlab.com/curben/phishing-filter.git phishing-filter/ - - git clone --depth 5 https://gitlab.com/curben/pup-filter.git pup-filter/ - - - mkdir -p oisd/ - - curl -L https://abp.oisd.nl/basic/ -o oisd/oisd_abp_light.txt - - curl -L https://abp.oisd.nl/ -o oisd/oisd_abp.txt - - curl -L https://dbl.oisd.nl/basic/ -o oisd/oisd_dbl_light.txt - - curl -L https://dbl.oisd.nl/ -o oisd/oisd_dbl.txt - - curl -L https://dblw.oisd.nl/basic/ -o oisd/oisd_dblw_light.txt - - curl -L https://dblw.oisd.nl/ -o oisd/oisd_dblw.txt - - curl -L https://hosts.oisd.nl/basic/ -o oisd/oisd_hosts_light.txt - - curl -L https://hosts.oisd.nl/ -o oisd/oisd_hosts.txt - - curl -L https://dnsmasq.oisd.nl/basic/ -o oisd/oisd_dnsmasq_light.txt - - curl -L https://dnsmasq.oisd.nl/ -o oisd/oisd_dnsmasq.txt - - curl -L https://rpz.oisd.nl/basic/ -o oisd/oisd_rpz_light.txt - - curl -L https://rpz.oisd.nl/ -o oisd/oisd_rpz.txt - - - mkdir -p tracking-filter/ - - git clone --depth 5 https://gitlab.com/curben/tracking-filter.git tracking-filter/ + - apk update && apk add brotli curl script: - - mkdir -p public/ - - cp urlhaus-filter/urlhaus-filter* public/ - - cp phishing-filter/dist/phishing-filter* public/ - - cp pup-filter/dist/pup-filter* public/ - - cp oisd/* public/ - - cp tracking-filter/dist/* public/ - - find public -type f -regex '.*\.\(txt\|conf\|tpl\|rules\)$' -exec gzip -f -k -9 {} \; - - find public -type f -regex '.*\.\(txt\|conf\|tpl\|rules\)$' -exec brotli -f -k -9 {} \; + - sh src/script.sh artifacts: paths: - - public/ + - tmp + - public + +pages: + stage: + - deploy + + script: + - echo + + artifacts: + paths: + - public rules: - - if: '$CI_COMMIT_REF_NAME == "master" && ($CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "web" || $CI_PIPELINE_SOURCE == "schedule")' - when: always + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH diff --git a/package.json b/package.json new file mode 100644 index 0000000..eb6d69a --- /dev/null +++ b/package.json @@ -0,0 +1,14 @@ +{ + "name": "malware-filter", + "private": true, + "scripts": { + "build": "node src/build.js" + }, + "dependencies": { + "extract-zip": "^2.0.1", + "got": "^11.8.3" + }, + "engines": { + "node": ">= 14.15.0" + } +} diff --git a/src/build.js b/src/build.js new file mode 100644 index 0000000..ed8834a --- /dev/null +++ b/src/build.js @@ -0,0 +1,30 @@ +'use strict' + +// for deployment outside of GitLab CI, e.g. Cloudflare Pages and Netlify + +const { stream: gotStream } = require('got') +const unzip = require('extract-zip') +const { join } = require('path') +const { mkdir } = require('fs/promises') +const { createWriteStream } = require('fs') +const { pipeline } = require('stream/promises') + +const rootPath = join(__dirname, '..') +const tmpPath = join(rootPath, 'tmp') +const zipPath = join(tmpPath, 'artifacts.zip') +const artifactsUrl = 'https://gitlab.com/curben/malware-filter/-/jobs/artifacts/main/download?job=pages' + +const f = async () => { + await mkdir(tmpPath, { recursive: true }) + + console.log(`Downloading artifacts.zip from "${artifactsUrl}"`) + await pipeline( + gotStream(artifactsUrl), + createWriteStream(zipPath) + ) + + console.log('Extracting artifacts.zip...') + await unzip(zipPath, { dir: rootPath }) +} + +f() diff --git a/src/script.sh b/src/script.sh new file mode 100644 index 0000000..665240d --- /dev/null +++ b/src/script.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +set -efux -o pipefail + +alias cp="cp -f" +alias curl="curl -L" +alias mkdir="mkdir -p" +alias unzip="unzip -jo" + +mkdir "tmp/" +cd "tmp/" + +curl "https://gitlab.com/curben/urlhaus-filter/-/jobs/artifacts/main/download?job=pages" -o "urlhaus-filter.zip" +curl "https://gitlab.com/curben/phishing-filter/-/jobs/artifacts/main/download?job=pages" -o "phishing-filter.zip" +curl "https://gitlab.com/curben/pup-filter/-/jobs/artifacts/main/download?job=pages" -o "pup-filter.zip" +curl "https://gitlab.com/curben/tracking-filter/-/jobs/artifacts/main/download?job=pages" -o "tracking-filter.zip" + +for zipfile in $(find . -name "*.zip" -type f) +do + unzip "$zipfile" +done + +curl "https://abp.oisd.nl/basic/" -o "oisd_abp_light.txt" +curl "https://abp.oisd.nl/" -o "oisd_abp.txt" +curl "https://dbl.oisd.nl/basic/" -o "oisd_dbl_light.txt" +curl "https://dbl.oisd.nl/" -o "oisd_dbl.txt" +curl "https://dblw.oisd.nl/basic/" -o "oisd_dblw_light.txt" +curl "https://dblw.oisd.nl/" -o "oisd_dblw.txt" +curl "https://hosts.oisd.nl/basic/" -o "oisd_hosts_light.txt" +curl "https://hosts.oisd.nl/" -o "oisd_hosts.txt" +curl "https://dnsmasq.oisd.nl/basic/" -o "oisd_dnsmasq_light.txt" +curl "https://dnsmasq.oisd.nl/" -o "oisd_dnsmasq.txt" +curl "https://rpz.oisd.nl/basic/" -o "oisd_rpz_light.txt" +curl "https://rpz.oisd.nl/" -o "oisd_rpz.txt" + +find . -name "oisd*.txt" -type f -print0 | xargs -0 gzip -f -k -9 +find . -name "oisd*.txt" -type f -print0 | xargs -0 brotli -f -k -9 + +mkdir "../public/" +find . ! -name "*.zip" -type f | xargs cp -t "../public/"