fix: extract from file instead of stream

incomplete extraction
https://github.com/ZJONSSON/node-unzipper/issues/335

when using stream in Node 22.14, botnet-filter.txt(.br|.gz) are missing
in bun 1.2.5, botnet-filter.txt is extracted but not .br/gz

revert 499330a12d93cb30c5771354316ddade260ebfdc7
This commit is contained in:
MDLeom 2025-03-23 02:31:11 +00:00
parent 61a25e8992
commit c79644a2f5
No known key found for this signature in database
GPG Key ID: 32D3E28E96A695E8
1 changed files with 16 additions and 4 deletions

View File

@ -7,12 +7,13 @@
// Instead of using the API, I find it easier to failover to GitHub.
// ref: https://gitlab.com/gitlab-org/gitlab/-/issues/29257
import { Extract } from 'unzipper'
import { dirname, join } from 'node:path'
import { Open } from 'unzipper'
import { basename, dirname, join } from 'node:path'
import { mkdir, readdir, rm, stat } from 'node:fs/promises'
import { pipeline } from 'node:stream/promises'
import { fileURLToPath } from 'node:url'
import { Readable } from 'node:stream'
import { createWriteStream } from 'node:fs'
const __dirname = dirname(fileURLToPath(import.meta.url))
const rootPath = join(__dirname, '..')
@ -39,14 +40,16 @@ const pipelineStatus = async (url) => {
const dl = async (project) => {
const filename = project + '.zip'
const zipPath = join(tmpPath, filename)
const link = `https://gitlab.com/malware-filter/${project}/-/jobs/artifacts/main/download?job=pages`
const pipelineUrl = `https://gitlab.com/malware-filter/${project}/badges/main/pipeline.svg`
let isMirror = false
console.log(`Downloading ${filename} from "${link}"`)
try {
await pipeline(
Readable.fromWeb((await fetch(link)).body),
Extract({ path: rootPath })
createWriteStream(zipPath)
)
await pipelineStatus(pipelineUrl)
} catch ({ message }) {
@ -58,11 +61,12 @@ const dl = async (project) => {
const mirrorLink = `https://nightly.link/curbengh/${project}/workflows/pages/main/public.zip`
console.log(`Downloading ${filename} from "${mirrorLink}"`)
isMirror = true
try {
await pipeline(
Readable.fromWeb((await fetch(mirrorLink)).body),
Extract({ path: publicPath })
createWriteStream(zipPath)
)
} catch ({ message }) {
throw new Error(JSON.stringify({
@ -72,6 +76,14 @@ const dl = async (project) => {
}))
}
}
console.log(`Extracting ${basename(zipPath)}...`)
if (isMirror === false) {
await Open.file(zipPath).then((dir) => dir.extract({ path: rootPath }))
} else {
await Open.file(zipPath).then((dir) => dir.extract({ path: publicPath }))
}
}
const f = async () => {