125 lines
3.5 KiB
JavaScript
125 lines
3.5 KiB
JavaScript
'use strict'
|
|
|
|
const { stream: gotStream } = require('got')
|
|
const got = require('got')
|
|
const unzip = require('extract-zip')
|
|
const { basename, join } = require('path')
|
|
const { mkdir } = require('fs/promises')
|
|
const { createWriteStream } = require('fs')
|
|
const { pipeline } = require('stream/promises')
|
|
const envVar = process.env
|
|
|
|
const rootPath = join(__dirname, '..')
|
|
const tmpPath = join(rootPath, 'tmp')
|
|
const publicPath = join(rootPath, 'public')
|
|
const projects = [
|
|
'urlhaus-filter',
|
|
'phishing-filter',
|
|
'pup-filter',
|
|
'tracking-filter',
|
|
'vn-badsite-filter'
|
|
]
|
|
const oisdFilters = {
|
|
'https://abp.oisd.nl/basic/': 'oisd_abp_light.txt',
|
|
'https://abp.oisd.nl/': 'oisd_abp.txt',
|
|
'https://dbl.oisd.nl/basic/': 'oisd_dbl_light.txt',
|
|
'https://dbl.oisd.nl/': 'oisd_dbl.txt',
|
|
'https://dblw.oisd.nl/': 'oisd_dblw_light.txt',
|
|
'https://dblw.oisd.nl/basic/': 'oisd_dblw.txt',
|
|
'https://hosts.oisd.nl/basic/': 'oisd_hosts_light.txt',
|
|
'https://hosts.oisd.nl/': 'oisd_hosts.txt',
|
|
'https://dnsmasq.oisd.nl/basic/': 'oisd_dnsmasq_light.txt',
|
|
'https://dnsmasq.oisd.nl/': 'oisd_dnsmasq.txt',
|
|
'https://rpz.oisd.nl/basic/': 'oisd_rpz_light.txt',
|
|
'https://rpz.oisd.nl/': 'oisd_rpz.txt',
|
|
}
|
|
|
|
const pipelineStatus = async (url) => {
|
|
try {
|
|
const svg = await got(url).text()
|
|
if (!svg.includes('passed')) throw new Error('last gitlab pipeline failed')
|
|
} catch ({ message }) {
|
|
throw new Error(message)
|
|
}
|
|
}
|
|
|
|
const dl = async (project) => {
|
|
const filename = project + '.zip'
|
|
const link = `https://gitlab.com/malware-filter/${project}/-/jobs/artifacts/main/download?job=pages`
|
|
const zipPath = join(tmpPath, filename)
|
|
const pipelineUrl = `https://gitlab.com/malware-filter/${project}/badges/main/pipeline.svg`
|
|
let isMirror = false
|
|
|
|
console.log(`Downloading ${filename} from "${link}"`)
|
|
try {
|
|
await pipeline(
|
|
gotStream(link),
|
|
createWriteStream(zipPath)
|
|
)
|
|
await pipelineStatus(pipelineUrl)
|
|
} catch ({ message }) {
|
|
console.error(JSON.stringify({
|
|
error: message,
|
|
link,
|
|
filename
|
|
}))
|
|
|
|
const mirrorLink = `https://nightly.link/curbengh/${project}/workflows/pages/main/public.zip`
|
|
console.log(`Downloading ${filename} from "${mirrorLink}"`)
|
|
isMirror = true
|
|
|
|
try {
|
|
await pipeline(
|
|
gotStream(mirrorLink),
|
|
createWriteStream(zipPath)
|
|
)
|
|
} catch ({ message }) {
|
|
throw new Error(JSON.stringify({
|
|
error: message,
|
|
link: mirrorLink,
|
|
filename
|
|
}))
|
|
}
|
|
}
|
|
|
|
console.log(`Extracting ${basename(zipPath)}...`)
|
|
if (isMirror === false) {
|
|
await unzip(zipPath, { dir: rootPath })
|
|
} else {
|
|
await unzip(zipPath, { dir: publicPath })
|
|
}
|
|
}
|
|
|
|
const oisdDl = async (link, filename) => {
|
|
const txtPath = join(publicPath, filename)
|
|
console.log(`Downloading ${filename} from "${link}"`)
|
|
try {
|
|
await pipeline(
|
|
gotStream(link),
|
|
createWriteStream(txtPath)
|
|
)
|
|
} catch ({ message }) {
|
|
console.error(JSON.stringify({
|
|
error: message,
|
|
link,
|
|
filename
|
|
}))
|
|
}
|
|
}
|
|
|
|
const f = async () => {
|
|
await mkdir(tmpPath, { recursive: true })
|
|
await mkdir(publicPath, { recursive: true })
|
|
await Promise.all(projects.map((project) => { return dl(project) }))
|
|
}
|
|
|
|
const oisd = async () => {
|
|
await mkdir(publicPath, { recursive: true })
|
|
if (Object.prototype.hasOwnProperty.call(envVar, 'CI_PROJECT_PATH') && envVar.CI_PROJECT_PATH === 'malware-filter/malware-filter') {
|
|
await Promise.all(Object.entries(oisdFilters).map(([link, filename]) => { return oisdDl(link, filename) }))
|
|
}
|
|
}
|
|
|
|
f()
|
|
oisd()
|