diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4dc66f0..6d69a84 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: alpine:latest +image: node:lts-alpine build_job: stage: build @@ -45,10 +45,13 @@ netlify: stage: deploy before_script: - - apk update && apk add curl + - npm install + - npm install netlify-cli -g + - netlify --telemetry-disable script: - - curl -X POST -d "{}" "https://api.netlify.com/build_hooks/$NETLIFY_BUILD_HOOK" + - npm run build + - netlify deploy --dir=public rules: - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $NETLIFY == "true"' diff --git a/src/build.js b/src/build.js index 4436e5d..6b2bef8 100644 --- a/src/build.js +++ b/src/build.js @@ -4,33 +4,43 @@ const { stream: gotStream } = require('got') const unzip = require('extract-zip') -const { join } = require('path') -const { mkdir, readdir, rm } = require('fs/promises') +const { basename, join } = require('path') +const { mkdir } = require('fs/promises') const { createWriteStream } = require('fs') const { pipeline } = require('stream/promises') const rootPath = join(__dirname, '..') const tmpPath = join(rootPath, 'tmp') -const zipPath = join(tmpPath, 'artifacts.zip') -const artifactsUrl = 'https://gitlab.com/curben/malware-filter/-/jobs/artifacts/main/download?job=pages' -const publicPath = join(rootPath, 'public') +const artifacts = { + 'https://gitlab.com/curben/urlhaus-filter/-/jobs/artifacts/main/download?job=pages': 'urlhaus-filter.zip', + 'https://gitlab.com/curben/phishing-filter/-/jobs/artifacts/main/download?job=pages': 'phishing-filter.zip', + 'https://gitlab.com/curben/pup-filter/-/jobs/artifacts/main/download?job=pages': 'pup-filter.zip', + 'https://gitlab.com/curben/tracking-filter/-/jobs/artifacts/main/download?job=pages': 'tracking-filter.zip' +} + +const dl = async (link, filename) => { + const zipPath = join(tmpPath, filename) + console.log(`Downloading ${filename} from "${link}"`) + try { + await pipeline( + gotStream(link), + createWriteStream(zipPath) + ) + } catch ({ message }) { + throw new Error(JSON.stringify({ + error: message, + link, + filename + })) + } + + console.log(`Extracting ${basename(zipPath)}...`) + await unzip(zipPath, { dir: rootPath }) +} const f = async () => { await mkdir(tmpPath, { recursive: true }) - - console.log(`Downloading artifacts.zip from "${artifactsUrl}"`) - await pipeline( - gotStream(artifactsUrl), - createWriteStream(zipPath) - ) - - console.log('Extracting artifacts.zip...') - await unzip(zipPath, { dir: rootPath }) - - const files = await readdir(publicPath) - for (const file of files) { - if (file.startsWith('oisd')) await rm(join(publicPath, file)) - } + await Promise.all(Object.entries(artifacts).map(([link, filename]) => { return dl(link, filename) })) } f()