From 7f2033e55352be68767ce65466c14c64356c1df3 Mon Sep 17 00:00:00 2001 From: MDLeom <43627182+curbengh@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:06:11 +0000 Subject: [PATCH 1/3] feat: zstd compression disabled by default due to lack of server support close #156 --- README.md | 29 +++++- index.js | 9 ++ lib/filter.js | 35 +++++++ package.json | 3 +- test/zstd.test.js | 256 ++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 330 insertions(+), 2 deletions(-) create mode 100644 test/zstd.test.js diff --git a/README.md b/README.md index 73cb0f4..a0308f2 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![NPM Dependencies](https://img.shields.io/librariesio/release/npm/hexo-yam)](https://libraries.io/npm/hexo-yam) [![Known Vulnerabilities](https://img.shields.io/snyk/vulnerabilities/npm/hexo-yam?logo=snyk)](https://snyk.io/test/npm/hexo-yam) -Yet Another Minifier for Hexo. Minify and compress HTML, JS, CSS, SVG, XML and JSON. [Other files](https://github.com/curbengh/hexo-yam/blob/ba77db0094a7c07ea9f70f010bfc15541d4105ca/index.js#L64) are also compressed. Support gzip and [brotli](https://en.wikipedia.org/wiki/Brotli) [compressions](https://en.wikipedia.org/wiki/HTTP_compression). +Yet Another Minifier for Hexo. Minify and compress HTML, JS, CSS, SVG, XML and JSON. [Other files](https://github.com/curbengh/hexo-yam/blob/ba77db0094a7c07ea9f70f010bfc15541d4105ca/index.js#L64) are also compressed. Support gzip, brotli and zstd [compressions](https://en.wikipedia.org/wiki/HTTP_compression). ## Table of contents @@ -19,6 +19,7 @@ Yet Another Minifier for Hexo. Minify and compress HTML, JS, CSS, SVG, XML and J - [SVG](#svg) - [Gzip](#gzip) - [Brotli](#brotli) +- [Zstd](#zstd) - [XML](#xml) - [JSON](#json) - [Globbing](#globbing) @@ -190,6 +191,32 @@ minify: - **globOptions** - See [globbing](#globbing) section. - **level** - Compression level. Range `1-11`. Defaults to `11`, or the value of [`zlib.constants.BROTLI_MAX_QUALITY`](https://nodejs.org/docs/latest-v12.x/api/zlib.html#zlib_brotli_constants) +## Zstd + +``` yaml +minify: + zstd: + enable: false + include: + - '*.html' + - '*.css' + - '*.js' + - '*.txt' + - '*.ttf' + - '*.atom' + - '*.stl' + - '*.xml' + - '*.svg' + - '*.eot' + - '*.json' +``` +- **enable** - Enable the plugin. Defaults to `false`. +- **priority** - Plugin's priority. Defaults to `10`. +- **verbose** - Verbose output. Defaults to `false`. +- **include** - Include files. Support [wildcard](http://www.globtester.com/) pattern(s) in a string or array. +- **globOptions** - See [globbing](#globbing) section. +- **level** - Compression level. Range `1-22`. Defaults to `3`, or the value of [`DEFAULT_LEVEL`](https://github.com/mongodb-js/zstd/blob/a3a08c61c9045411c8275e248498dbc583457fb5/src/lib.rs#L9) + ## XML Remove whitespaces in xml. diff --git a/index.js b/index.js index 318533a..1d6cd94 100644 --- a/index.js +++ b/index.js @@ -68,6 +68,14 @@ hexo.config.minify.brotli = Object.assign({ globOptions: { basename: true } }, hexo.config.minify.brotli) +hexo.config.minify.zstd = Object.assign({ + enable: false, + priority: 10, + verbose: false, + include: ['*.html', '*.css', '*.js', '*.txt', '*.ttf', '*.atom', '*.stl', '*.xml', '*.svg', '*.eot', '*.json'], + globOptions: { basename: true } +}, hexo.config.minify.zstd) + hexo.config.minify.xml = Object.assign({ enable: false, priority: 10, @@ -93,6 +101,7 @@ if (hexo.config.minify.enable === true) { hexo.extend.filter.register('after_generate', filter.minifySvg, hexo.config.minify.svg.priority) hexo.extend.filter.register('after_generate', filter.gzipFn, hexo.config.minify.gzip.priority) hexo.extend.filter.register('after_generate', filter.brotliFn, hexo.config.minify.brotli.priority) + hexo.extend.filter.register('after_generate', filter.zstdFn, hexo.config.minify.zstd.priority) hexo.extend.filter.register('after_generate', filter.minifyXml, hexo.config.minify.xml.priority) hexo.extend.filter.register('after_generate', filter.minifyJson, hexo.config.minify.json.priority) } diff --git a/lib/filter.js b/lib/filter.js index 48b3df4..0d17003 100644 --- a/lib/filter.js +++ b/lib/filter.js @@ -10,6 +10,7 @@ const gzip = promisify(zlib.gzip) const br = promisify(zlib.brotliCompress) const { minify: compressXml } = require('minify-xml') const micromatch = require('micromatch') +const { compress: zstd } = require('@mongodb-js/zstd') const isMatch = (path = '', patterns = [], options = {}) => { if (path && patterns) { @@ -230,6 +231,39 @@ function brotliFn () { })) } +function zstdFn () { + const hexo = this + const options = hexo.config.minify.zstd + if (options.enable === false) return + + const { route } = hexo + const routeList = route.list() + const { globOptions, include, verbose } = options + let { level } = options + if (typeof level !== 'number') level = undefined + + return Promise.all((match(routeList, include, globOptions)).map((path) => { + return new Promise((resolve, reject) => { + const assetPath = route.get(path) + let assetTxt = '' + assetPath.on('data', (chunk) => (assetTxt += chunk)) + assetPath.on('end', async () => { + if (assetTxt.length) { + try { + const input = Buffer.from(assetTxt, 'utf-8') + const result = await zstd(input, level) + if (verbose) logFn.call(this, assetTxt, result, path, 'zstd') + resolve(route.set(path + '.zst', result)) + } catch (err) { + reject(new Error(`Path: ${path}\n${err}`)) + } + } + resolve() + }) + }) + })) +} + function minifyXml () { const hexo = this const options = hexo.config.minify.xml @@ -297,6 +331,7 @@ module.exports = { minifySvg, gzipFn, brotliFn, + zstdFn, minifyXml, minifyJson } diff --git a/package.json b/package.json index 0807395..52cad19 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,8 @@ "micromatch": "^4.0.2", "minify-xml": "^3.2.0", "svgo": "^3.0.0", - "terser": "^5.3.0" + "terser": "^5.3.0", + "@mongodb-js/zstd": "^1.2.0" }, "devDependencies": { "hexo": "^7.1.0", diff --git a/test/zstd.test.js b/test/zstd.test.js new file mode 100644 index 0000000..c6e236e --- /dev/null +++ b/test/zstd.test.js @@ -0,0 +1,256 @@ +/* eslint-env jest */ +'use strict' + +const Hexo = require('hexo') +const { compress: zstd, decompress: unzstd } = require('@mongodb-js/zstd') + +describe('zstd', () => { + const hexo = new Hexo(__dirname) + const z = require('../lib/filter').zstdFn.bind(hexo) + const path = 'foo.txt' + const input = 'Lorem ipsum dolor sit amet consectetur adipiscing elit fusce' + const inputBuf = Buffer.from(input, 'utf8') + + beforeEach(() => { + hexo.config.minify = { + zstd: { + enable: true, + verbose: false, + include: ['*.html', '*.css', '*.js', '*.txt', '*.ttf', '*.atom', '*.stl', '*.xml', '*.svg', '*.eot', '*.json'], + globOptions: { basename: true } + } + } + hexo.route.set(path, input) + }) + + afterEach(() => { + const routeList = hexo.route.list() + routeList.forEach((path) => hexo.route.remove(path)) + }) + + test('default', async () => { + await z() + + const output = hexo.route.get(path.concat('.zst')) + const buf = [] + output.on('data', (chunk) => (buf.push(chunk))) + output.on('end', async () => { + const result = Buffer.concat(buf) + const expected = await zstd(inputBuf) + const resultUnzst = await unzstd(result) + const expectedUnzst = await unzstd(expected) + + expect(result.equals(expected)).toBe(true) + expect(resultUnzst.toString()).toBe(input) + expect(expectedUnzst.toString()).toBe(input) + }) + }) + + test('disable', async () => { + hexo.config.minify.zstd.enable = false + const result = await z() + + expect(result).toBeUndefined() + }) + + test('empty file', async () => { + hexo.route.set(path, '') + + const routeList = hexo.route.list() + expect(routeList).not.toContain(path.concat('.zst')) + + const result = await z() + expect(result).toBeDefined() + expect(result[0]).toBeUndefined() + }) + + test('option', async () => { + const level = 1 + hexo.config.minify.zstd.level = level + await z() + + const output = hexo.route.get(path.concat('.zst')) + const buf = [] + output.on('data', (chunk) => (buf.push(chunk))) + output.on('end', async () => { + const result = Buffer.concat(buf) + const expected = await zstd(inputBuf, level) + + expect(result.equals(expected)).toBe(true) + }) + }) + + test('option - verbose', async () => { + hexo.config.minify.zstd.verbose = true + hexo.log.log = jest.fn() + await z() + + expect(hexo.log.log.mock.calls[0][0]).toContain(`zstd: ${path}`) + }) + + test('option - level is string', async () => { + const level = 'foo' + hexo.config.minify.zstd.level = level + await z() + + const output = hexo.route.get(path.concat('.zst')) + const buf = [] + output.on('data', (chunk) => (buf.push(chunk))) + output.on('end', async () => { + const result = Buffer.concat(buf) + const expected = await zstd(inputBuf, undefined) + + expect(result.equals(expected)).toBe(true) + }) + }) + + test('include - exclude non-text file by default', async () => { + const path = 'foo.jpg' + hexo.route.set(path, input) + await z() + + const result = hexo.route.get(path.concat('.zst')) + + expect(result).toBeUndefined() + }) + + test('include - basename', async () => { + hexo.config.minify.zstd.include = 'bar.txt' + const path = 'foo/bar.txt' + hexo.route.set(path, input) + await z() + + const result = hexo.route.get(path.concat('.zst')) + + expect(result).toBeDefined() + }) + + test('include - slash in pattern', async () => { + hexo.config.minify.zstd.include = '**/lectus/**/*.txt' + const path = 'eleifend/lectus/nullam/dapibus/netus.txt' + hexo.route.set(path, input) + await z() + + const result = hexo.route.get(path.concat('.zst')) + + expect(result).toBeDefined() + }) + + test('include - basename + slash + basename enabled', async () => { + hexo.route.remove(path) + + const paths = [ + 'lorem/ipsum/dolor.html', + 'gravida/sociis/erat/ante.css', + 'aptent/elementum.js', + 'felis/blandit/cursus.svg' + ] + hexo.config.minify.zstd.include = [ + '*.html', + '**/sociis/**/*.css' + ] + + paths.forEach((inpath) => { + hexo.route.set(inpath, input) + }) + await z() + + const routeList = hexo.route.list() + const expected = [ + 'lorem/ipsum/dolor.html.zst', + 'gravida/sociis/erat/ante.css.zst' + ] + const notExpected = [ + 'aptent/elementum.js.zst', + 'felis/blandit/cursus.svg.zst' + ] + + expect(routeList).toEqual(expect.arrayContaining(expected)) + expect(routeList).toEqual(expect.not.arrayContaining(notExpected)) + }) + + test('include - basename + slash + basename disabled', async () => { + hexo.route.remove(path) + + const paths = [ + 'lorem/ipsum/dolor.html', + 'gravida/sociis/erat/ante.css', + 'aptent/elementum.js', + 'felis/blandit/cursus.svg' + ] + hexo.config.minify.zstd.include = [ + '*.html', + '**/sociis/**/*.css' + ] + hexo.config.minify.zstd.globOptions = { + basename: false + } + + paths.forEach((inpath) => { + hexo.route.set(inpath, input) + }) + await z() + + const routeList = hexo.route.list() + const expected = [ + 'gravida/sociis/erat/ante.css.zst' + ] + const notExpected = [ + 'lorem/ipsum/dolor.html.zst', + 'aptent/elementum.js.zst', + 'felis/blandit/cursus.svg.zst' + ] + + expect(routeList).toEqual(expect.arrayContaining(expected)) + expect(routeList).toEqual(expect.not.arrayContaining(notExpected)) + }) + + test('include - reverse pattern + basename disabled', async () => { + hexo.route.remove(path) + + const paths = [ + 'lorem/ipsum/dolor.html', + 'gravida/sociis/erat/ante.css', + 'aptent/elementum.js', + 'felis/blandit/cursus.svg' + ] + hexo.config.minify.zstd.include = [ + '!dolor.html' + ] + hexo.config.minify.zstd.globOptions = { + basename: false + } + + paths.forEach((inpath) => { + hexo.route.set(inpath, input) + }) + await z() + + const routeList = hexo.route.list() + const expected = paths.map((path) => path.concat('.zst')) + + expect(routeList).toEqual(expect.arrayContaining(expected)) + }) + + test('blns', async () => { + const blns = require('./fixtures/blns.json') + + for (const nStr of blns) { + hexo.route.remove(path) + + hexo.route.set(path, nStr) + + await z() + + const output = hexo.route.get(path.concat('.zst')) + const buf = [] + output.on('data', (chunk) => (buf.push(chunk))) + output.on('end', async () => { + const result = Buffer.concat(buf) + const resultUnzst = await unzstd(result) + + expect(resultUnzst.toString()).toBe(nStr) + }) + } + }) +}) From 18a5d8108bd8ac0080a8244498ff3b5524ed5ad1 Mon Sep 17 00:00:00 2001 From: MDLeom <43627182+curbengh@users.noreply.github.com> Date: Thu, 6 Jun 2024 11:41:34 +0000 Subject: [PATCH 2/3] test: include optional deps to include zstd binary --- .github/workflows/tester.yml | 2 +- README.md | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tester.yml b/.github/workflows/tester.yml index 1c97a46..8ccbce9 100644 --- a/.github/workflows/tester.yml +++ b/.github/workflows/tester.yml @@ -23,7 +23,7 @@ jobs: key: ${{ runner.os }}-npm-cache restore-keys: ${{ runner.os }}-npm-cache - name: Install Dependencies - run: npm install + run: npm install --include=optional - name: Test run: npm run test env: diff --git a/README.md b/README.md index a0308f2..29fc259 100644 --- a/README.md +++ b/README.md @@ -217,6 +217,10 @@ minify: - **globOptions** - See [globbing](#globbing) section. - **level** - Compression level. Range `1-22`. Defaults to `3`, or the value of [`DEFAULT_LEVEL`](https://github.com/mongodb-js/zstd/blob/a3a08c61c9045411c8275e248498dbc583457fb5/src/lib.rs#L9) +### Cannot find module '@mongodb-js/zstd-linux-x64-gnu' + +`npm install --include=optional` + ## XML Remove whitespaces in xml. From 1b04f7c8f1c80d35e6906f748fc3e3d5925a840f Mon Sep 17 00:00:00 2001 From: MDLeom <43627182+curbengh@users.noreply.github.com> Date: Fri, 7 Jun 2024 09:04:05 +0000 Subject: [PATCH 3/3] docs(zstd): --force flag to install optional deps of child deps https://stackoverflow.com/a/42400102 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 29fc259..8ff7193 100644 --- a/README.md +++ b/README.md @@ -219,7 +219,7 @@ minify: ### Cannot find module '@mongodb-js/zstd-linux-x64-gnu' -`npm install --include=optional` +`npm install --include=optional --force` ## XML