Finalize v0.6.0

CHANGELOG.md

@@ -1,5 +1,8 @@
 ## 0.6.0
 - Revamped streaming unzip for compatibility and performance improvements
+- Fixed streaming data bugs
+- Fixed inflation errors
+- Planned new tests
 ## 0.5.2
 - General bugfixes
 ## 0.5.0

README.md

@@ -40,7 +40,7 @@ import * as fflate from 'fflate';
 // However, you should import ONLY what you need to minimize bloat.
 // So, if you just need GZIP compression support:
 import { gzipSync } from 'fflate';
-// Woo! You just saved 15 kB off your bundle with one line.
+// Woo! You just saved 20 kB off your bundle with one line.
 ```
 
 If your environment doesn't support ES Modules (e.g. Node.js):
@@ -57,13 +57,18 @@ You should use either UNPKG or jsDelivr (i.e. only one of the following)
 
 Note that tree shaking is completely unsupported from the CDN. If you want
 a small build without build tools, please ask me and I will make one manually
-with only the features you need.
+with only the features you need. This build is about 27kB, or 9kB gzipped.
 
 You may also want to specify the version, e.g. with [email protected]
 -->
 <script src="https://unpkg.com/fflate/umd/index.js"></script>
 <script src="https://cdn.jsdelivr.net/npm/fflate/umd/index.js"></script>
 <!-- Now, the global variable fflate contains the library -->
+
+<!-- If you're going buildless but want ESM, import from Skypack -->
+<script type="module">
+  import * as fflate from 'https://cdn.skypack.dev/fflate?min';
+</script>
 ```
 
 If your environment doesn't support bundling:
@@ -373,7 +378,7 @@ if (needToCancel) {
 // If you wish to provide options, use the second argument.
 
 // The consume option will render the data inside aMassiveFile unusable,
-// but can dramatically improve performance and reduce memory usage.
+// but can improve performance and dramatically reduce memory usage.
 zlib(aMassiveFile, { consume: true, level: 9 }, (err, data) => {
   // Use the data
 });

package.json

@@ -62,7 +62,7 @@
   "scripts": {
     "build": "yarn build:lib && yarn build:docs && yarn build:demo",
     "script": "node -r ts-node/register scripts/$SC.ts",
-    "build:lib": "tsc && tsc --project tsconfig.esm.json && yarn build:umd && yarn build:rewrite",
+    "build:lib": "tsc && tsc --project tsconfig.esm.json && yarn build:rewrite && yarn build:umd",
     "build:umd": "SC=buildUMD yarn script",
     "build:rewrite": "SC=rewriteBuilds yarn script",
     "build:demo": "tsc --project tsconfig.demo.json && parcel build demo/index.html --public-url \"./\" && SC=cpGHPages yarn script",

scripts/buildUMD.ts

@@ -28,12 +28,9 @@ minify(src, opts).then(async out => {
   const nodeWkrOut = (await minify(nodeWorker, opts)).code!.replace(
     /exports.__esModule=!0;/,
     ''
-  ).replace(/exports\./g, '_f.').replace(
-    /require/, // intentionally done just once
-    "eval('require')"
   );
   const res = "!function(f){typeof module!='undefined'&&typeof exports=='object'?module.exports=f():typeof define!='undefined'&&define.amd?define(['fflate',f]):(typeof self!='undefined'?self:this).fflate=f()}(function(){var _e={};" +
-    out.code!.replace(/exports\.(.*)=void 0;/, '').replace(/exports\./g, '_e.').replace(/require\("\.\/node-worker"\)/,
+    out.code!.replace(/exports\.(.*) = void 0;\n/, '').replace(/exports\./g, '_e.').replace(/require\("\.\/node-worker"\)/,
     "(typeof module!='undefined'&&typeof exports=='object'?function(_f){" + nodeWkrOut + 'return _f}:function(_f){' + wkrOut + 'return _f})({})'
   ) + 'return _e})';
   if (!existsSync(p('umd'))) mkdirSync(p('umd'));

src/index.ts

@@ -174,6 +174,7 @@ type InflateState = {
 const inflt = (dat: Uint8Array, buf?: Uint8Array, st?: InflateState) => {
   // source length
   const sl = dat.length;
+  if (!sl || (st && !st.l && sl < 5)) return buf || new u8(0);
   // have to estimate size
   const noBuf = !buf || (st as unknown as boolean);
   // no state
@@ -934,8 +935,7 @@ const cbify = <T extends AsyncOptions>(dat: Uint8Array, opts: T, fns: (() => unk
       cb(err, dat);
     }
   );
-  if (!opts.consume) dat = new u8(dat);
-  w.postMessage([dat, opts], [dat.buffer]);
+  w.postMessage([dat, opts], opts.consume ? [dat.buffer] : []);
   return () => { w.terminate(); };
 }
 
@@ -1441,7 +1441,7 @@ export class Gunzip {
   push(chunk: Uint8Array, final?: boolean) {
     (Inflate.prototype as unknown as { e: typeof Inflate.prototype['e'] }).e.call(this, chunk);
     if (this.v) {
-      const s = gzs(this.p);
+      const s = this.p.length > 3 ? gzs(this.p) : 4;
       if (s >= this.p.length && !final) return;
       this.p = this.p.subarray(s), this.v = 0;
     }

test/2-perf.ts

@@ -20,6 +20,7 @@ const cache: Record<string, Record<string, Buffer>> = {
 const flattenedWorkers: Record<string, TestHandler> = {};
 for (const k in workers) {
   for (const l in workers[k]) {
+    if (l == 'zip' || l == 'unzip') continue;
     flattenedWorkers[k + '.' + l] = async (file, name, resetTimer) => {
       const fileClone = bClone(file);
       let buf = fileClone;

test/3-streams.ts

@@ -1 +0,0 @@
-// TODO: Test streams here

test/3-zip.ts

@@ -0,0 +1 @@
+// TODO: test ZIP

test/4-streams.ts

@@ -0,0 +1 @@
+// TODO: test all streams (including ZIP)

test/5-async.ts

@@ -0,0 +1 @@
+// TODO: test all async operations (including streams and ZIP)

test/util.ts

@@ -14,7 +14,9 @@ const testFiles = {
 };
 
 const testZipFiles = {
-
+  model3D: 'https://raw.githubusercontent.com/mrdoob/three.js/dev/examples/models/kmz/Box.kmz',
+  largeModel3D: 'https://raw.githubusercontent.com/mrdoob/three.js/dev/examples/models/3mf/truck.3mf',
+  repo: 'https://codeload.github.com/parcel-bundler/parcel/zip/v2'
 };
 
 const dlCached = async <T extends Record<string, string | Buffer>>(files: T) => {
@@ -50,22 +52,25 @@ const dlCached = async <T extends Record<string, string | Buffer>>(files: T) =>
   return res;
 }
 
-export type TestFile = keyof typeof testFiles;
-
 const testFilesPromise = dlCached(testFiles);
+const testZipFilesPromise = dlCached(testZipFiles);
+
 export type TestHandler = (file: Buffer, name: string, resetTimer: () => void) => unknown | Promise<unknown>;
 
-export const testSuites = async <T extends Record<string, TestHandler>>(suites: T) => {
-  const perf = {} as Record<keyof T, Promise<Record<TestFile, number>>>;
+export const testSuites = async <T extends Record<string, TestHandler>, D extends 'zip' | 'default' = 'default'>(suites: T, type?: D) => {
+  type DK = keyof (D extends 'zip' ? typeof testZipFiles : typeof testFiles);
+  const tf = type == 'zip' ? testZipFiles : testFiles;
+  const tfp = type == 'zip' ? testZipFilesPromise : testFilesPromise;
+  const perf = {} as Record<keyof T, Promise<Record<DK, number>>>;
   for (const k in suites) {
     perf[k] = new Promise(async setPerf => {
       const ste = suite(k);
-      let localTestFiles: Record<TestFile, Buffer>;
-      ste.before(() => testFilesPromise.then(v => {
-        localTestFiles = v;
-      }));
-      const localPerf: Record<string, number> = {};
-      for (const name in testFiles) {
+      let localTestFiles: Record<DK, Buffer>;
+      ste.before(async () => {
+        localTestFiles = (await tfp) as unknown as Record<DK, Buffer>;
+      });
+      const localPerf = {} as Record<DK, number>;
+      for (const name in tf) {
         ste(name, async () => {
           let ts = performance.now();
           await suites[k](localTestFiles[name], name, () => {
@@ -80,7 +85,7 @@ export const testSuites = async <T extends Record<string, TestHandler>>(suites:
       ste.run();
     })
   }
-  const resolvedPerf = {} as Record<keyof T, Record<TestFile, number>>;
+  const resolvedPerf = {} as Record<keyof T, Record<DK, number>>;
   for (const k in suites) resolvedPerf[k] = await perf[k];
   return resolvedPerf;
 };
@@ -88,8 +93,8 @@ export const testSuites = async <T extends Record<string, TestHandler>>(suites:
 export const stream = (src: Uint8Array, dst: {
   push(dat: Uint8Array, final: boolean): void;
 }) => {
-  const off = Math.min(65536, src.length >>> 3);
   for (let i = 0; i < src.length;) {
+    const off = Math.floor(Math.random() * Math.min(131072, src.length >>> 3));
     dst.push(src.slice(i, i + off), (i += off) >= src.length);
   }
 }
@@ -106,24 +111,14 @@ const cws = (pkg: string, method: string = '_cjsDefault') => `
   }
 `;
 
-// create callback worker string
-const cbws = (pkg: string, method: string, alias = 'run') => `
-  const { ${method}: ${alias} } = require('${pkg}');
-  const { Worker, workerData, parentPort } = require('worker_threads');
-  ${alias}(...(Array.isArray(workerData) ? workerData : [workerData]), (err, buf) => {
-    if (err) parentPort.postMessage({ err });
-    else parentPort.postMessage(buf, [buf.buffer]);
-  });
-`;
-
 export type Workerized = (workerData: Uint8Array | [Uint8Array, {}], transferable?: ArrayBuffer[]) => WorkerizedResult;
 export interface WorkerizedResult extends PromiseLike<Uint8Array> {
   timeout(ms: number): void;
 };
 
 // Worker creator
-const wc = (pkg: string, method?: string, cb = false): Workerized => {
-  const str = cb ? cbws(pkg, method) : cws(pkg, method);
+const wc = (pkg: string, method?: string): Workerized => {
+  const str = cws(pkg, method);
   return (workerData, transferable) => {
     const worker = new Worker(str, {
       eval: true,
@@ -166,7 +161,9 @@ export const workers = {
     gzip: wc(fflate, 'gzipSync'),
     gunzip: wc(fflate, 'gunzipSync'),
     zlib: wc(fflate, 'zlibSync'),
-    unzlib: wc(fflate, 'unzlibSync')
+    unzlib: wc(fflate, 'unzlibSync'),
+    zip: wc(fflate, 'zipSync'),
+    unzip: wc(fflate, 'unzipSync')
   },
   pako: {
     deflate: wc('pako', 'deflateRaw'),
@@ -184,12 +181,12 @@ export const workers = {
     inflate: wc('tiny-inflate')
   },
   zlib: {
-    deflate: wc('zlib', 'deflateRaw', true),
-    inflate: wc('zlib', 'inflateRaw', true),
-    gzip: wc('zlib', 'gzip', true),
-    gunzip: wc('zlib', 'gunzip', true),
-    zlib: wc('zlib', 'deflate', true),
-    unzlib: wc('zlib', 'inflate', true)
+    deflate: wc('zlib', 'deflateRawSync'),
+    inflate: wc('zlib', 'inflateRawSync'),
+    gzip: wc('zlib', 'gzipSync'),
+    gunzip: wc('zlib', 'gunzipSync'),
+    zlib: wc('zlib', 'deflateSync'),
+    unzlib: wc('zlib', 'inflateSync')
   }
 };