diff --git a/src/build.js b/src/build.js index 3985e11..d12c662 100644 --- a/src/build.js +++ b/src/build.js @@ -1,6 +1,7 @@ 'use strict' const { stream: gotStream } = require('got') +const got = require('got') const unzip = require('extract-zip') const { basename, join, parse: pathParse } = require('path') const { mkdir } = require('fs/promises') @@ -11,18 +12,12 @@ const envVar = process.env const rootPath = join(__dirname, '..') const tmpPath = join(rootPath, 'tmp') const publicPath = join(rootPath, 'public') -const artifacts = { - 'https://gitlab.com/curben/urlhaus-filter/-/jobs/artifacts/main/download?job=pages': 'urlhaus-filter.zip', - 'https://gitlab.com/curben/phishing-filter/-/jobs/artifacts/main/download?job=pages': 'phishing-filter.zip', - 'https://gitlab.com/curben/pup-filter/-/jobs/artifacts/main/download?job=pages': 'pup-filter.zip', - 'https://gitlab.com/curben/tracking-filter/-/jobs/artifacts/main/download?job=pages': 'tracking-filter.zip' -} -const ghMirror = { - 'urlhaus-filter': 'https://nightly.link/curbengh/urlhaus-filter/workflows/pages/main/public.zip', - 'phishing-filter': 'https://nightly.link/curbengh/phishing-filter/workflows/pages/main/public.zip', - 'pup-filter': 'https://nightly.link/curbengh/pup-filter/workflows/pages/main/public.zip', - 'tracking-filter': 'https://nightly.link/curbengh/tracking-filter/workflows/pages/main/public.zip' -} +const projects = [ + 'urlhaus-filter', + 'phishing-filter', + 'pup-filter', + 'tracking-filter' +] const oisdFilters = { 'https://abp.oisd.nl/basic/': 'oisd_abp_light.txt', 'https://abp.oisd.nl/': 'oisd_abp.txt', @@ -38,8 +33,20 @@ const oisdFilters = { 'https://rpz.oisd.nl/': 'oisd_rpz.txt', } -const dl = async (link, filename) => { +const pipelineStatus = async (url) => { + try { + const svg = await got(url).text() + if (!svg.includes('passed')) throw new Error('last gitlab pipeline failed') + } catch ({ message }) { + throw new Error(message) + } +} + +const dl = async (project) => { + const filename = project + '.zip' + const link = `https://gitlab.com/curben/${project}/-/jobs/artifacts/main/download?job=pages` const zipPath = join(tmpPath, filename) + const pipelineUrl = `https://gitlab.com/curben/${project}/badges/main/pipeline.svg` let isMirror = false console.log(`Downloading ${filename} from "${link}"`) @@ -48,6 +55,7 @@ const dl = async (link, filename) => { gotStream(link), createWriteStream(zipPath) ) + await pipelineStatus(pipelineUrl) } catch ({ message }) { console.error(JSON.stringify({ error: message, @@ -55,7 +63,7 @@ const dl = async (link, filename) => { filename })) - const mirrorLink = ghMirror[pathParse(filename).name] + const mirrorLink = `https://nightly.link/curbengh/${project}/workflows/pages/main/public.zip` console.log(`Downloading ${filename} from "${mirrorLink}"`) isMirror = true @@ -101,7 +109,7 @@ const oisdDl = async (link, filename) => { const f = async () => { await mkdir(tmpPath, { recursive: true }) await mkdir(publicPath, { recursive: true }) - await Promise.all(Object.entries(artifacts).map(([link, filename]) => { return dl(link, filename) })) + await Promise.all(projects.map((project) => { return dl(project) })) } const oisd = async () => { diff --git a/src/gitlab_status.sh b/src/gitlab_status.sh index 8e49e55..56095c0 100644 --- a/src/gitlab_status.sh +++ b/src/gitlab_status.sh @@ -1,9 +1,10 @@ #!/bin/sh -CURL_STATUS=$(curl -sSIL "https://gitlab.com/curben/urlhaus-filter/-/jobs/artifacts/main/download?job=pages" | grep -F "HTTP/2 200") +ARTIFACT_STATUS=$(curl -sSIL "https://gitlab.com/curben/urlhaus-filter/-/jobs/artifacts/main/download?job=pages" | grep -F "HTTP/2 200") +PIPELINE_STATUS=$(curl -sSL "https://gitlab.com/curben/urlhaus-filter/badges/main/pipeline.svg" | grep -F "failed") GITLAB_STATUS="up" -if [ -z "$CURL_STATUS" ]; then +if [ -z "$ARTIFACT_STATUS" ] || [ -n "$PIPELINE_STATUS" ]; then GITLAB_STATUS="down" fi