build: failover to github artifacts when gitlab is unavailable

- Closes #1

.gitlab-ci.yml

@@ -15,7 +15,7 @@ build_job:
     - apk update && apk add brotli curl
 
   script:
-    - sh src/script.sh
+    - npm run build
     - find public -name "oisd*.txt" -type f -print0 | xargs -0 gzip -f -k -9
     - find public -name "oisd*.txt" -type f -print0 | xargs -0 brotli -f -k -9
 

src/build.js

@@ -1,25 +1,47 @@
 'use strict'
 
-// for deployment outside of GitLab CI, e.g. Cloudflare Pages and Netlify
-
 const { stream: gotStream } = require('got')
 const unzip = require('extract-zip')
-const { basename, join } = require('path')
+const { basename, join, parse: pathParse } = require('path')
 const { mkdir } = require('fs/promises')
 const { createWriteStream } = require('fs')
 const { pipeline } = require('stream/promises')
+const envVar = process.env
 
 const rootPath = join(__dirname, '..')
 const tmpPath = join(rootPath, 'tmp')
+const publicPath = join(rootPath, 'public')
 const artifacts = {
   'https://gitlab.com/curben/urlhaus-filter/-/jobs/artifacts/main/download?job=pages': 'urlhaus-filter.zip',
   'https://gitlab.com/curben/phishing-filter/-/jobs/artifacts/main/download?job=pages': 'phishing-filter.zip',
   'https://gitlab.com/curben/pup-filter/-/jobs/artifacts/main/download?job=pages': 'pup-filter.zip',
   'https://gitlab.com/curben/tracking-filter/-/jobs/artifacts/main/download?job=pages': 'tracking-filter.zip'
 }
+const ghMirror = {
+  'urlhaus-filter': 'https://nightly.link/curbengh/urlhaus-filter/workflows/pages/main/public.zip',
+  'phishing-filter': 'https://nightly.link/curbengh/phishing-filter/workflows/pages/main/public.zip',
+  'pup-filter': 'https://nightly.link/curbengh/pup-filter/workflows/pages/main/public.zip',
+  'tracking-filter': 'https://nightly.link/curbengh/tracking-filter/workflows/pages/main/public.zip'
+}
+const oisdFilters = {
+  'https://abp.oisd.nl/basic/': 'oisd_abp_light.txt',
+  'https://abp.oisd.nl/': 'oisd_abp.txt',
+  'https://dbl.oisd.nl/basic/': 'oisd_dbl_light.txt',
+  'https://dbl.oisd.nl/': 'oisd_dbl.txt',
+  'https://dblw.oisd.nl/': 'oisd_dblw_light.txt',
+  'https://dblw.oisd.nl/basic/': 'oisd_dblw.txt',
+  'https://hosts.oisd.nl/basic/': 'oisd_hosts_light.txt',
+  'https://hosts.oisd.nl/': 'oisd_hosts.txt',
+  'https://dnsmasq.oisd.nl/basic/': 'oisd_dnsmasq_light.txt',
+  'https://dnsmasq.oisd.nl/': 'oisd_dnsmasq.txt',
+  'https://rpz.oisd.nl/basic/': 'oisd_rpz_light.txt',
+  'https://rpz.oisd.nl/': 'oisd_rpz.txt',
+}
 
 const dl = async (link, filename) => {
   const zipPath = join(tmpPath, filename)
+  let isMirror = false
+
   console.log(`Downloading ${filename} from "${link}"`)
   try {
     await pipeline(
@@ -27,20 +49,67 @@ const dl = async (link, filename) => {
       createWriteStream(zipPath)
     )
   } catch ({ message }) {
-    throw new Error(JSON.stringify({
+    console.error(JSON.stringify({
+      error: message,
+      link,
+      filename
+    }))
+
+    const mirrorLink = ghMirror[pathParse(filename).name]
+    console.log(`Downloading ${filename} from "${mirrorLink}"`)
+    isMirror = true
+
+    try {
+      await pipeline(
+        gotStream(mirrorLink),
+        createWriteStream(zipPath)
+      )
+    } catch ({ message }) {
+      throw new Error(JSON.stringify({
+        error: message,
+        link: mirrorLink,
+        filename
+      }))
+    }
+  }
+
+  console.log(`Extracting ${basename(zipPath)}...`)
+  if (isMirror === false) {
+    await unzip(zipPath, { dir: rootPath })
+  } else {
+    await unzip(zipPath, { dir: publicPath })
+  }
+}
+
+const oisdDl = async (link, filename) => {
+  const txtPath = join(publicPath, filename)
+  console.log(`Downloading ${filename} from "${link}"`)
+  try {
+    await pipeline(
+      gotStream(link),
+      createWriteStream(txtPath)
+    )
+  } catch ({ message }) {
+    console.error(JSON.stringify({
       error: message,
       link,
       filename
     }))
   }
-
-  console.log(`Extracting ${basename(zipPath)}...`)
-  await unzip(zipPath, { dir: rootPath })
 }
 
 const f = async () => {
   await mkdir(tmpPath, { recursive: true })
+  await mkdir(publicPath, { recursive: true })
   await Promise.all(Object.entries(artifacts).map(([link, filename]) => { return dl(link, filename) }))
 }
 
+const oisd = async () => {
+  await mkdir(publicPath, { recursive: true })
+  if (Object.prototype.hasOwnProperty.call(envVar, 'CI_PROJECT_PATH') && envVar.CI_PROJECT_PATH === 'curben/malware-filter') {
+    await Promise.all(Object.entries(oisdFilters).map(([link, filename]) => { return oisdDl(link, filename) }))
+  }
+}
+
 f()
+oisd()