malware-filter/src/build.js

'use strict'

// for deployment outside of GitLab CI, e.g. Cloudflare Pages and Netlify

const { stream: gotStream } = require('got')
const unzip = require('extract-zip')
const { basename, join } = require('path')
const { mkdir } = require('fs/promises')
const { createWriteStream } = require('fs')
const { pipeline } = require('stream/promises')

const rootPath = join(__dirname, '..')
const tmpPath = join(rootPath, 'tmp')
const artifacts = {
  'https://gitlab.com/curben/urlhaus-filter/-/jobs/artifacts/main/download?job=pages': 'urlhaus-filter.zip',
  'https://gitlab.com/curben/phishing-filter/-/jobs/artifacts/main/download?job=pages': 'phishing-filter.zip',
  'https://gitlab.com/curben/pup-filter/-/jobs/artifacts/main/download?job=pages': 'pup-filter.zip',
  'https://gitlab.com/curben/tracking-filter/-/jobs/artifacts/main/download?job=pages': 'tracking-filter.zip'
}

const dl = async (link, filename) => {
  const zipPath = join(tmpPath, filename)
  console.log(`Downloading ${filename} from "${link}"`)
  try {
    await pipeline(
      gotStream(link),
      createWriteStream(zipPath)
    )
  } catch ({ message }) {
    throw new Error(JSON.stringify({
      error: message,
      link,
      filename
    }))
  }

  console.log(`Extracting ${basename(zipPath)}...`)
  await unzip(zipPath, { dir: rootPath })
}

const f = async () => {
  await mkdir(tmpPath, { recursive: true })
  await Promise.all(Object.entries(artifacts).map(([link, filename]) => { return dl(link, filename) }))
}

f()