diff --git a/assembly/hash/md4.asm.ts b/assembly/hash/md4.asm.ts new file mode 100644 index 000000000..3388ecc57 --- /dev/null +++ b/assembly/hash/md4.asm.ts @@ -0,0 +1,177 @@ +/* + ** ******************************************************************** + ** md4.c -- Implementation of MD4 Message Digest Algorithm ** + ** Updated: 2/16/90 by Ronald L. Rivest ** + ** (C) 1990 RSA Data Security, Inc. ** + ** ******************************************************************** + */ + +// Ported to assemblyscript by Tobias Koppers + +let totalLength: u32; +let A: u32; +let B: u32; +let C: u32; +let D: u32; + +function F(x: u32, y: u32, z: u32): u32 { + return z ^ (x & (y ^ z)); +} +function G(x: u32, y: u32, z: u32): u32 { + return (x & (y | z)) | (y & z); +} +function H(x: u32, y: u32, z: u32): u32 { + return x ^ y ^ z; +} + +function roundF(a: u32, b: u32, c: u32, d: u32, i: u32, s: u32): u32 { + return rotl(a + F(b, c, d) + load(i), s); +} +function roundG(a: u32, b: u32, c: u32, d: u32, i: u32, s: u32): u32 { + return rotl(a + G(b, c, d) + load(i) + 0x5a827999, s); +} +function roundH(a: u32, b: u32, c: u32, d: u32, i: u32, s: u32): u32 { + return rotl(a + H(b, c, d) + load(i) + 0x6ed9eba1, s); +} + +export function init(): void { + A = 0x67452301; + B = 0xefcdab89; + C = 0x98badcfe; + D = 0x10325476; + totalLength = 0; +} + +function body(size: u32): void { + let _A = A; + let _B = B; + let _C = C; + let _D = D; + + for (let i: u32 = 0; i < size; i += 64) { + let a = _A; + let b = _B; + let c = _C; + let d = _D; + + // Round F + + a = roundF(a, b, c, d, i + 4 * 0, 3); + d = roundF(d, a, b, c, i + 4 * 1, 7); + c = roundF(c, d, a, b, i + 4 * 2, 11); + b = roundF(b, c, d, a, i + 4 * 3, 19); + + a = roundF(a, b, c, d, i + 4 * 4, 3); + d = roundF(d, a, b, c, i + 4 * 5, 7); + c = roundF(c, d, a, b, i + 4 * 6, 11); + b = roundF(b, c, d, a, i + 4 * 7, 19); + + a = roundF(a, b, c, d, i + 4 * 8, 3); + d = roundF(d, a, b, c, i + 4 * 9, 7); + c = roundF(c, d, a, b, i + 4 * 10, 11); + b = roundF(b, c, d, a, i + 4 * 11, 19); + + a = roundF(a, b, c, d, i + 4 * 12, 3); + d = roundF(d, a, b, c, i + 4 * 13, 7); + c = roundF(c, d, a, b, i + 4 * 14, 11); + b = roundF(b, c, d, a, i + 4 * 15, 19); + + // Round G + + a = roundG(a, b, c, d, i + 4 * 0, 3); + d = roundG(d, a, b, c, i + 4 * 4, 5); + c = roundG(c, d, a, b, i + 4 * 8, 9); + b = roundG(b, c, d, a, i + 4 * 12, 13); + + a = roundG(a, b, c, d, i + 4 * 1, 3); + d = roundG(d, a, b, c, i + 4 * 5, 5); + c = roundG(c, d, a, b, i + 4 * 9, 9); + b = roundG(b, c, d, a, i + 4 * 13, 13); + + a = roundG(a, b, c, d, i + 4 * 2, 3); + d = roundG(d, a, b, c, i + 4 * 6, 5); + c = roundG(c, d, a, b, i + 4 * 10, 9); + b = roundG(b, c, d, a, i + 4 * 14, 13); + + a = roundG(a, b, c, d, i + 4 * 3, 3); + d = roundG(d, a, b, c, i + 4 * 7, 5); + c = roundG(c, d, a, b, i + 4 * 11, 9); + b = roundG(b, c, d, a, i + 4 * 15, 13); + + // Round H + + a = roundH(a, b, c, d, i + 4 * 0, 3); + d = roundH(d, a, b, c, i + 4 * 8, 9); + c = roundH(c, d, a, b, i + 4 * 4, 11); + b = roundH(b, c, d, a, i + 4 * 12, 15); + + a = roundH(a, b, c, d, i + 4 * 2, 3); + d = roundH(d, a, b, c, i + 4 * 10, 9); + c = roundH(c, d, a, b, i + 4 * 6, 11); + b = roundH(b, c, d, a, i + 4 * 14, 15); + + a = roundH(a, b, c, d, i + 4 * 1, 3); + d = roundH(d, a, b, c, i + 4 * 9, 9); + c = roundH(c, d, a, b, i + 4 * 5, 11); + b = roundH(b, c, d, a, i + 4 * 13, 15); + + a = roundH(a, b, c, d, i + 4 * 3, 3); + d = roundH(d, a, b, c, i + 4 * 11, 9); + c = roundH(c, d, a, b, i + 4 * 7, 11); + b = roundH(b, c, d, a, i + 4 * 15, 15); + + _A += a; + _B += b; + _C += c; + _D += d; + } + + A = _A; + B = _B; + C = _C; + D = _D; +} + +export function update(length: u32): void { + body(length); + totalLength += length; +} + +export function final(length: u32): void { + const bits: u64 = u64(totalLength + length) << 3; + const finalLength: u32 = (length + 9 + 63) & ~63; + const bitsPosition = finalLength - 8; + + // end + store(length++, 0x80); + + // padding + for (; length & 7 && length < finalLength; length++) store(length, 0); + for (; length < finalLength; length += 8) store(length, 0); + + // bits + store(bitsPosition, bits); + + body(finalLength); + + store(0, u32ToHex(A)); + store(8, u32ToHex(B)); + store(16, u32ToHex(C)); + store(24, u32ToHex(D)); +} + +function u32ToHex(x: u64): u64 { + // from https://johnnylee-sde.github.io/Fast-unsigned-integer-to-hex-string/ + + x = ((x & 0xffff0000) << 16) | (x & 0xffff); + x = ((x & 0x0000ff000000ff00) << 8) | (x & 0x000000ff000000ff); + x = ((x & 0x00f000f000f000f0) >> 4) | ((x & 0x000f000f000f000f) << 8); + + const mask = ((x + 0x0606060606060606) >> 4) & 0x0101010101010101; + + x |= 0x3030303030303030; + + x += 0x27 * mask; + + return x; +} diff --git a/assembly/hash/xxhash64.asm.ts b/assembly/hash/xxhash64.asm.ts index 3525a7720..7f6b9df43 100644 --- a/assembly/hash/xxhash64.asm.ts +++ b/assembly/hash/xxhash64.asm.ts @@ -108,8 +108,6 @@ export function final(length: u32): void { result *= Prime3; result ^= result >> 32; - store(0, result); - store(0, u32ToHex(result >> 32)); store(8, u32ToHex(result & 0xffffffff)); } diff --git a/benchmark/md4.js b/benchmark/md4.js new file mode 100644 index 000000000..e302fdfc1 --- /dev/null +++ b/benchmark/md4.js @@ -0,0 +1,87 @@ +const crypto = require("crypto"); +const createHash = require("../lib/util/createHash"); + +let result; + +const measure = (fn, count) => { + const start = process.hrtime.bigint(); + for (let i = 0; i < count; i++) result = fn(); + return Number(process.hrtime.bigint() - start); +}; + +const NS_PER_MS = 1000000; // 1ms +const MIN_DURATION = 100 * NS_PER_MS; // 100ms +const MAX_DURATION = 1000 * NS_PER_MS; // 1000ms +const MAX_WARMUP_DURATION = 1 * NS_PER_MS; // 1ms + +const format = (fast, slow, fastName, slowName, count) => { + return `${fastName} is ${ + Math.round(((slow - fast) * 1000) / slow) / 10 + }% faster than ${slowName} (${Math.round(fast / 100 / count) / 10} µs vs ${ + Math.round(slow / 100 / count) / 10 + } µs, ${count}x)`; +}; + +const compare = (n1, f1, n2, f2) => { + let count = 1; + while (true) { + const timings = [f1, f2, f1, f2, f1, f2].map(f => measure(f, count)); + const t1 = Math.min(timings[0], timings[2], timings[4]); + const t2 = Math.min(timings[1], timings[3], timings[5]); + if (count === 1 && (t1 > MAX_WARMUP_DURATION || t2 > MAX_WARMUP_DURATION)) { + continue; + } + if ( + (t1 > MIN_DURATION && t2 > MIN_DURATION) || + t1 > MAX_DURATION || + t2 > MAX_DURATION + ) { + return t1 > t2 + ? format(t2, t1, n2, n1, count) + : format(t1, t2, n1, n2, count); + } + count *= 2; + } +}; + +for (const size of [ + 1, 2, 4, 8, 10, 20, 40, 60, 80, 100, 200, 1000, 5000, 10000, 20000, 50000, + 100000, 200000, 500000 +]) { + const longString = require("crypto").randomBytes(size).toString("hex"); + const buffer = require("crypto").randomBytes(size * 2); + console.log( + `string ${longString.length} chars: ` + + compare( + "crypto md4", + () => { + const hash = crypto.createHash("md4"); + hash.update(longString); + return hash.digest("hex"); + }, + "wasm md4", + () => { + const hash = createHash("md4"); + hash.update(longString); + return hash.digest("hex"); + } + ) + ); + console.log( + `buffer ${buffer.length} bytes: ` + + compare( + "crypto md4", + () => { + const hash = crypto.createHash("md4"); + hash.update(buffer); + return hash.digest("hex"); + }, + "wasm md4", + () => { + const hash = createHash("md4"); + hash.update(buffer); + return hash.digest("hex"); + } + ) + ); +} diff --git a/declarations/LoaderContext.d.ts b/declarations/LoaderContext.d.ts index b7564e169..077f53b10 100644 --- a/declarations/LoaderContext.d.ts +++ b/declarations/LoaderContext.d.ts @@ -5,6 +5,7 @@ import type { ResolveOptionsWithDependencyType } from "../lib/ResolverFactory"; import type Compilation from "../lib/Compilation"; import type Compiler from "../lib/Compiler"; import type NormalModule from "../lib/NormalModule"; +import type Hash from "../lib/util/Hash"; import type { InputFileSystem } from "../lib/util/fs"; import type { Logger } from "../lib/logging/Logger"; import type { @@ -39,6 +40,7 @@ export interface NormalModuleLoaderContext { utils: { absolutify: (context: string, request: string) => string; contextify: (context: string, request: string) => string; + createHash: (algorithm: string) => Hash; }; rootContext: string; fs: InputFileSystem; diff --git a/lib/NormalModule.js b/lib/NormalModule.js index a917d66e0..a9bb4999d 100644 --- a/lib/NormalModule.js +++ b/lib/NormalModule.js @@ -538,7 +538,8 @@ class NormalModule extends Module { return context === this.context ? getContextifyInContext()(request) : getContextify()(context, request); - } + }, + createHash }; const loaderContext = { version: 2, diff --git a/lib/util/createHash.js b/lib/util/createHash.js index bd8d2f0e2..f727a1fdc 100644 --- a/lib/util/createHash.js +++ b/lib/util/createHash.js @@ -126,6 +126,7 @@ class DebugHash extends Hash { let crypto = undefined; let createXXHash64 = undefined; +let createMd4 = undefined; let BatchedHash = undefined; /** @@ -149,6 +150,17 @@ module.exports = algorithm => { } } return new BatchedHash(createXXHash64()); + case "md4": + if (createMd4 === undefined) { + createMd4 = require("./hash/md4"); + if (BatchedHash === undefined) { + BatchedHash = require("./hash/BatchedHash"); + } + } + return new BatchedHash(createMd4()); + case "native-md4": + if (crypto === undefined) crypto = require("crypto"); + return new BulkUpdateDecorator(() => crypto.createHash("md4"), "md4"); default: if (crypto === undefined) crypto = require("crypto"); return new BulkUpdateDecorator( diff --git a/lib/util/hash/md4.js b/lib/util/hash/md4.js new file mode 100644 index 000000000..6baaae85c --- /dev/null +++ b/lib/util/hash/md4.js @@ -0,0 +1,20 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +const create = require("./wasm-hash"); + +//#region wasm code: md4 (../../../assembly/hash/md4.asm.ts) --initialMemory 1 +const md4 = new WebAssembly.Module( + Buffer.from( + // 2150 bytes + "AGFzbQEAAAABCAJgAX8AYAAAAwUEAQAAAAUDAQABBhoFfwFBAAt/AUEAC38BQQALfwFBAAt/AUEACwciBARpbml0AAAGdXBkYXRlAAIFZmluYWwAAwZtZW1vcnkCAAqFEAQmAEGBxpS6BiQBQYnXtv5+JAJB/rnrxXkkA0H2qMmBASQEQQAkAAvMCgEYfyMBIQojAiEGIwMhByMEIQgDQCAAIAVLBEAgBSgCCCINIAcgBiAFKAIEIgsgCCAHIAUoAgAiDCAKIAggBiAHIAhzcXNqakEDdyIDIAYgB3Nxc2pqQQd3IgEgAyAGc3FzampBC3chAiAFKAIUIg8gASACIAUoAhAiCSADIAEgBSgCDCIOIAYgAyACIAEgA3Nxc2pqQRN3IgQgASACc3FzampBA3ciAyACIARzcXNqakEHdyEBIAUoAiAiEiADIAEgBSgCHCIRIAQgAyAFKAIYIhAgAiAEIAEgAyAEc3FzampBC3ciAiABIANzcXNqakETdyIEIAEgAnNxc2pqQQN3IQMgBSgCLCIVIAQgAyAFKAIoIhQgAiAEIAUoAiQiEyABIAIgAyACIARzcXNqakEHdyIBIAMgBHNxc2pqQQt3IgIgASADc3FzampBE3chBCAPIBAgCSAVIBQgEyAFKAI4IhYgAiAEIAUoAjQiFyABIAIgBSgCMCIYIAMgASAEIAEgAnNxc2pqQQN3IgEgAiAEc3FzampBB3ciAiABIARzcXNqakELdyIDIAkgAiAMIAEgBSgCPCIJIAQgASADIAEgAnNxc2pqQRN3IgEgAiADcnEgAiADcXJqakGZ84nUBWpBA3ciAiABIANycSABIANxcmpqQZnzidQFakEFdyIEIAEgAnJxIAEgAnFyaiASakGZ84nUBWpBCXciAyAPIAQgCyACIBggASADIAIgBHJxIAIgBHFyampBmfOJ1AVqQQ13IgEgAyAEcnEgAyAEcXJqakGZ84nUBWpBA3ciAiABIANycSABIANxcmpqQZnzidQFakEFdyIEIAEgAnJxIAEgAnFyampBmfOJ1AVqQQl3IgMgECAEIAIgFyABIAMgAiAEcnEgAiAEcXJqakGZ84nUBWpBDXciASADIARycSADIARxcmogDWpBmfOJ1AVqQQN3IgIgASADcnEgASADcXJqakGZ84nUBWpBBXciBCABIAJycSABIAJxcmpqQZnzidQFakEJdyIDIBEgBCAOIAIgFiABIAMgAiAEcnEgAiAEcXJqakGZ84nUBWpBDXciASADIARycSADIARxcmpqQZnzidQFakEDdyICIAEgA3JxIAEgA3FyampBmfOJ1AVqQQV3IgQgASACcnEgASACcXJqakGZ84nUBWpBCXciAyAMIAIgAyAJIAEgAyACIARycSACIARxcmpqQZnzidQFakENdyIBcyAEc2pqQaHX5/YGakEDdyICIAQgASACcyADc2ogEmpBodfn9gZqQQl3IgRzIAFzampBodfn9gZqQQt3IgMgAiADIBggASADIARzIAJzampBodfn9gZqQQ93IgFzIARzaiANakGh1+f2BmpBA3ciAiAUIAQgASACcyADc2pqQaHX5/YGakEJdyIEcyABc2pqQaHX5/YGakELdyIDIAsgAiADIBYgASADIARzIAJzampBodfn9gZqQQ93IgFzIARzampBodfn9gZqQQN3IgIgEyAEIAEgAnMgA3NqakGh1+f2BmpBCXciBHMgAXNqakGh1+f2BmpBC3chAyAKIA4gAiADIBcgASADIARzIAJzampBodfn9gZqQQ93IgFzIARzampBodfn9gZqQQN3IgJqIQogBiAJIAEgESADIAIgFSAEIAEgAnMgA3NqakGh1+f2BmpBCXciBHMgAXNqakGh1+f2BmpBC3ciAyAEcyACc2pqQaHX5/YGakEPd2ohBiADIAdqIQcgBCAIaiEIIAVBQGshBQwBCwsgCiQBIAYkAiAHJAMgCCQECw0AIAAQASMAIABqJAAL/wQCA38BfiMAIABqrUIDhiEEIABByABqQUBxIgJBCGshAyAAIgFBAWohACABQYABOgAAA0AgACACSUEAIABBB3EbBEAgAEEAOgAAIABBAWohAAwBCwsDQCAAIAJJBEAgAEIANwMAIABBCGohAAwBCwsgAyAENwMAIAIQAUEAIwGtIgRC//8DgyAEQoCA/P8Pg0IQhoQiBEL/gYCA8B+DIARCgP6DgIDgP4NCCIaEIgRCj4C8gPCBwAeDQgiGIARC8IHAh4CegPgAg0IEiIQiBEKGjJiw4MCBgwZ8QgSIQoGChIiQoMCAAYNCJ34gBEKw4MCBg4aMmDCEfDcDAEEIIwKtIgRC//8DgyAEQoCA/P8Pg0IQhoQiBEL/gYCA8B+DIARCgP6DgIDgP4NCCIaEIgRCj4C8gPCBwAeDQgiGIARC8IHAh4CegPgAg0IEiIQiBEKGjJiw4MCBgwZ8QgSIQoGChIiQoMCAAYNCJ34gBEKw4MCBg4aMmDCEfDcDAEEQIwOtIgRC//8DgyAEQoCA/P8Pg0IQhoQiBEL/gYCA8B+DIARCgP6DgIDgP4NCCIaEIgRCj4C8gPCBwAeDQgiGIARC8IHAh4CegPgAg0IEiIQiBEKGjJiw4MCBgwZ8QgSIQoGChIiQoMCAAYNCJ34gBEKw4MCBg4aMmDCEfDcDAEEYIwStIgRC//8DgyAEQoCA/P8Pg0IQhoQiBEL/gYCA8B+DIARCgP6DgIDgP4NCCIaEIgRCj4C8gPCBwAeDQgiGIARC8IHAh4CegPgAg0IEiIQiBEKGjJiw4MCBgwZ8QgSIQoGChIiQoMCAAYNCJ34gBEKw4MCBg4aMmDCEfDcDAAs=", + "base64" + ) +); +//#endregion + +module.exports = create.bind(null, md4, [], 64, 32); diff --git a/lib/util/hash/wasm-hash.js b/lib/util/hash/wasm-hash.js new file mode 100644 index 000000000..7d595cd46 --- /dev/null +++ b/lib/util/hash/wasm-hash.js @@ -0,0 +1,153 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +class WasmHash { + /** + * @param {WebAssembly.Instance} instance wasm instance + * @param {WebAssembly.Instance[]} instancesPool pool of instances + * @param {number} chunkSize size of data chunks passed to wasm + * @param {number} digestSize size of digest returned by wasm + */ + constructor(instance, instancesPool, chunkSize, digestSize) { + const exports = /** @type {any} */ (instance.exports); + exports.init(); + this.exports = exports; + this.mem = Buffer.from(exports.memory.buffer, 0, 65536); + this.buffered = 0; + this.instancesPool = instancesPool; + this.chunkSize = chunkSize; + this.digestSize = digestSize; + } + + reset() { + this.buffered = 0; + this.exports.init(); + } + + /** + * @param {Buffer | string} data data + * @param {BufferEncoding=} encoding encoding + * @returns {this} itself + */ + update(data, encoding) { + if (typeof data === "string") { + if (data.length < 21845) { + this._updateWithShortString(data, encoding); + return this; + } else { + data = Buffer.from(data, encoding); + } + } + this._updateWithBuffer(data); + return this; + } + + /** + * @param {string} data data + * @param {BufferEncoding=} encoding encoding + * @returns {void} + */ + _updateWithShortString(data, encoding) { + const { exports, buffered, mem, chunkSize } = this; + let endPos; + if (data.length < 70) { + if (!encoding || encoding === "utf-8" || encoding === "utf8") { + endPos = buffered; + for (let i = 0; i < data.length; i++) { + const cc = data.charCodeAt(i); + if (cc < 0x80) mem[endPos++] = cc; + else if (cc < 0x800) { + mem[endPos] = (cc >> 6) | 0xc0; + mem[endPos + 1] = (cc & 0x3f) | 0x80; + endPos += 2; + } else { + // bail-out for weird chars + endPos += mem.write(data.slice(endPos), endPos, encoding); + break; + } + } + } else if (encoding === "latin1") { + endPos = buffered; + for (let i = 0; i < data.length; i++) { + const cc = data.charCodeAt(i); + mem[endPos++] = cc; + } + } else { + endPos = buffered + mem.write(data, buffered, encoding); + } + } else { + endPos = buffered + mem.write(data, buffered, encoding); + } + if (endPos < chunkSize) { + this.buffered = endPos; + } else { + const l = endPos & ~(this.chunkSize - 1); + exports.update(l); + const newBuffered = endPos - l; + this.buffered = newBuffered; + if (newBuffered > 0) mem.copyWithin(0, l, endPos); + } + } + + /** + * @param {Buffer} data data + * @returns {void} + */ + _updateWithBuffer(data) { + const { exports, buffered, mem } = this; + const length = data.length; + if (buffered + length < this.chunkSize) { + data.copy(mem, buffered, 0, length); + this.buffered += length; + } else { + const l = (buffered + length) & ~(this.chunkSize - 1); + if (l > 65536) { + let i = 65536 - buffered; + data.copy(mem, buffered, 0, i); + exports.update(65536); + const stop = l - buffered - 65536; + while (i < stop) { + data.copy(mem, 0, i, i + 65536); + exports.update(65536); + i += 65536; + } + data.copy(mem, 0, i, l - buffered); + exports.update(l - buffered - i); + } else { + data.copy(mem, buffered, 0, l - buffered); + exports.update(l); + } + const newBuffered = length + buffered - l; + this.buffered = newBuffered; + if (newBuffered > 0) data.copy(mem, 0, length - newBuffered, length); + } + } + + digest(type) { + const { exports, buffered, mem, digestSize } = this; + exports.final(buffered); + this.instancesPool.push(this); + return mem.toString("latin1", 0, digestSize); + } +} + +const create = (wasmModule, instancesPool, chunkSize, digestSize) => { + if (instancesPool.length > 0) { + const old = instancesPool.pop(); + old.reset(); + return old; + } else { + return new WasmHash( + new WebAssembly.Instance(wasmModule), + instancesPool, + chunkSize, + digestSize + ); + } +}; + +module.exports = create; diff --git a/lib/util/hash/xxhash64.js b/lib/util/hash/xxhash64.js index 3bf2d3a30..ebd658ceb 100644 --- a/lib/util/hash/xxhash64.js +++ b/lib/util/hash/xxhash64.js @@ -5,150 +5,16 @@ "use strict"; +const create = require("./wasm-hash"); + //#region wasm code: xxhash64 (../../../assembly/hash/xxhash64.asm.ts) --initialMemory 1 const xxhash64 = new WebAssembly.Module( Buffer.from( - // 1180 bytes - "AGFzbQEAAAABCAJgAX8AYAAAAwQDAQAABQMBAAEGGgV+AUIAC34BQgALfgFCAAt+AUIAC34BQgALByIEBGluaXQAAAZ1cGRhdGUAAQVmaW5hbAACBm1lbW9yeQIACrwIAzAAQtbrgu7q/Yn14AAkAELP1tO+0ser2UIkAUIAJAJC+erQ0OfJoeThACQDQgAkBAvUAQIBfwR+IABFBEAPCyMEIACtfCQEIwAhAiMBIQMjAiEEIwMhBQNAIAIgASkDAELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiECIAMgASkDCELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEDIAQgASkDEELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEEIAUgASkDGELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEFIAAgAUEgaiIBSw0ACyACJAAgAyQBIAQkAiAFJAMLsgYCAX8EfiMEQgBSBH4jACICQgGJIwEiA0IHiXwjAiIEQgyJfCMDIgVCEol8IAJCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0gA0LP1tO+0ser2UJ+Qh+JQoeVr6+Ytt6bnn9+hUKHla+vmLbem55/fkKdo7Xqg7GNivoAfSAEQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IAVCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0FQsXP2bLx5brqJwsjBCAArXx8IQIDQCABQQhqIABNBEAgAiABKQMAQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQhuJQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IQIgAUEIaiEBDAELCyABQQRqIABNBEACfyACIAE1AgBCh5Wvr5i23puef36FQheJQs/W077Sx6vZQn5C+fPd8Zn2masWfCECIAFBBGoLIQELA0AgACABRwRAIAIgATEAAELFz9my8eW66id+hUILiUKHla+vmLbem55/fiECIAFBAWohAQwBCwtBACACIAJCIYiFQs/W077Sx6vZQn4iAiACQh2IhUL5893xmfaZqxZ+IgIgAkIgiIUiAjcDAEEAIAJCIIgiA0L//wODQiCGIANCgID8/w+DQhCIhCIDQv+BgIDwH4NCEIYgA0KA/oOAgOA/g0IIiIQiA0KPgLyA8IHAB4NCCIYgA0LwgcCHgJ6A+ACDQgSIhCIDQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiADQrDgwIGDhoyYMIR8NwMAQQggAkL/////D4MiAkL//wODQiCGIAJCgID8/w+DQhCIhCICQv+BgIDwH4NCEIYgAkKA/oOAgOA/g0IIiIQiAkKPgLyA8IHAB4NCCIYgAkLwgcCHgJ6A+ACDQgSIhCICQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiACQrDgwIGDhoyYMIR8NwMACw==", + // 1173 bytes + "AGFzbQEAAAABCAJgAX8AYAAAAwQDAQAABQMBAAEGGgV+AUIAC34BQgALfgFCAAt+AUIAC34BQgALByIEBGluaXQAAAZ1cGRhdGUAAQVmaW5hbAACBm1lbW9yeQIACrUIAzAAQtbrgu7q/Yn14AAkAELP1tO+0ser2UIkAUIAJAJC+erQ0OfJoeThACQDQgAkBAvUAQIBfwR+IABFBEAPCyMEIACtfCQEIwAhAiMBIQMjAiEEIwMhBQNAIAIgASkDAELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiECIAMgASkDCELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEDIAQgASkDEELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEEIAUgASkDGELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEFIAAgAUEgaiIBSw0ACyACJAAgAyQBIAQkAiAFJAMLqwYCAX8EfiMEQgBSBH4jACICQgGJIwEiA0IHiXwjAiIEQgyJfCMDIgVCEol8IAJCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0gA0LP1tO+0ser2UJ+Qh+JQoeVr6+Ytt6bnn9+hUKHla+vmLbem55/fkKdo7Xqg7GNivoAfSAEQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IAVCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0FQsXP2bLx5brqJwsjBCAArXx8IQIDQCABQQhqIABNBEAgAiABKQMAQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQhuJQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IQIgAUEIaiEBDAELCyABQQRqIABNBEACfyACIAE1AgBCh5Wvr5i23puef36FQheJQs/W077Sx6vZQn5C+fPd8Zn2masWfCECIAFBBGoLIQELA0AgACABRwRAIAIgATEAAELFz9my8eW66id+hUILiUKHla+vmLbem55/fiECIAFBAWohAQwBCwtBACACIAJCIYiFQs/W077Sx6vZQn4iAiACQh2IhUL5893xmfaZqxZ+IgIgAkIgiIUiAkIgiCIDQv//A4NCIIYgA0KAgPz/D4NCEIiEIgNC/4GAgPAfg0IQhiADQoD+g4CA4D+DQgiIhCIDQo+AvIDwgcAHg0IIhiADQvCBwIeAnoD4AINCBIiEIgNChoyYsODAgYMGfEIEiEKBgoSIkKDAgAGDQid+IANCsODAgYOGjJgwhHw3AwBBCCACQv////8PgyICQv//A4NCIIYgAkKAgPz/D4NCEIiEIgJC/4GAgPAfg0IQhiACQoD+g4CA4D+DQgiIhCICQo+AvIDwgcAHg0IIhiACQvCBwIeAnoD4AINCBIiEIgJChoyYsODAgYMGfEIEiEKBgoSIkKDAgAGDQid+IAJCsODAgYOGjJgwhHw3AwAL", "base64" ) ); //#endregion -class XxHash64 { - /** - * @param {WebAssembly.Instance} instance wasm instance - */ - constructor(instance) { - const exports = /** @type {any} */ (instance.exports); - exports.init(); - this.exports = exports; - this.mem = Buffer.from(exports.memory.buffer, 0, 65536); - this.buffered = 0; - } - - reset() { - this.buffered = 0; - this.exports.init(); - } - - /** - * @param {Buffer | string} data data - * @param {BufferEncoding=} encoding encoding - * @returns {this} itself - */ - update(data, encoding) { - if (typeof data === "string") { - if (data.length < 21845) { - this._updateWithShortString(data, encoding); - return this; - } else { - data = Buffer.from(data, encoding); - } - } - this._updateWithBuffer(data); - return this; - } - - /** - * @param {string} data data - * @param {BufferEncoding=} encoding encoding - * @returns {void} - */ - _updateWithShortString(data, encoding) { - const { exports, buffered, mem } = this; - let endPos; - if (data.length < 70) { - if (!encoding || encoding === "utf-8" || encoding === "utf8") { - endPos = buffered; - for (let i = 0; i < data.length; i++) { - const cc = data.charCodeAt(i); - if (cc < 0x80) mem[endPos++] = cc; - else if (cc < 0x800) { - mem[endPos] = (cc >> 6) | 0xc0; - mem[endPos + 1] = (cc & 0x3f) | 0x80; - endPos += 2; - } else { - // bail-out for weird chars - endPos += mem.write(data.slice(endPos), endPos, encoding); - break; - } - } - } else if (encoding === "latin1") { - endPos = buffered; - for (let i = 0; i < data.length; i++) { - const cc = data.charCodeAt(i); - mem[endPos++] = cc; - } - } else { - endPos = buffered + mem.write(data, buffered, encoding); - } - } else { - endPos = buffered + mem.write(data, buffered, encoding); - } - if (endPos < 32) { - this.buffered = endPos; - } else { - const l = (endPos >> 5) << 5; - exports.update(l); - const newBuffered = endPos - l; - this.buffered = newBuffered; - if (newBuffered > 0) mem.copyWithin(0, l, endPos); - } - } - - /** - * @param {Buffer} data data - * @returns {void} - */ - _updateWithBuffer(data) { - const { exports, buffered, mem } = this; - const length = data.length; - if (buffered + length < 32) { - data.copy(mem, buffered, 0, length); - this.buffered += length; - } else { - const l = ((buffered + length) >> 5) << 5; - if (l > 65536) { - let i = 65536 - buffered; - data.copy(mem, buffered, 0, i); - exports.update(65536); - const stop = l - buffered - 65536; - while (i < stop) { - data.copy(mem, 0, i, i + 65536); - exports.update(65536); - i += 65536; - } - data.copy(mem, 0, i, l - buffered); - exports.update(l - buffered - i); - } else { - data.copy(mem, buffered, 0, l - buffered); - exports.update(l); - } - const newBuffered = length + buffered - l; - this.buffered = newBuffered; - if (newBuffered > 0) data.copy(mem, 0, length - newBuffered, length); - } - } - - digest(type) { - const { exports, buffered, mem } = this; - exports.final(buffered); - instancesPool.push(this); - return mem.toString("latin1", 0, 16); - } -} - -const instancesPool = []; - -const create = () => { - if (instancesPool.length > 0) { - const old = instancesPool.pop(); - old.reset(); - return old; - } else { - return new XxHash64(new WebAssembly.Instance(xxhash64)); - } -}; - -module.exports = create; +module.exports = create.bind(null, xxhash64, [], 32, 16); diff --git a/test/WasmHashes.unittest.js b/test/WasmHashes.unittest.js new file mode 100644 index 000000000..21a59351f --- /dev/null +++ b/test/WasmHashes.unittest.js @@ -0,0 +1,96 @@ +const { randomBytes, createHash } = require("crypto"); + +const wasmHashes = { + xxhash64: () => { + const createHash = require("../lib/util/hash/xxhash64"); + const createReferenceHash = + require("hash-wasm/dist/xxhash64.umd.min.js").createXXHash64; + return { + createHash, + createReferenceHash: async () => (await createReferenceHash()).init(), + regExp: /^[0-9a-f]{16}$/ + }; + }, + md4: () => { + const createMd4Hash = require("../lib/util/hash/md4"); + return { + createHash: createMd4Hash, + createReferenceHash: async () => createHash("md4"), + regExp: /^[0-9a-f]{32}$/ + }; + } +}; + +for (const name of Object.keys(wasmHashes)) { + const { createHash, createReferenceHash, regExp } = wasmHashes[name](); + + describe(name, () => { + const sizes = [ + 1, + 2, + 3, + 4, + 5, + 7, + 8, + 9, + 16, + 31, + 32, + 33, + 64 - 10, + 64 - 9, + 64 - 8, + 63, + 64, + 65, + 100, + 1000, + 65536 - 1, + 65536, + 65536 + 1, + 65536 + 31, + 65536 * 5, + 65536 * 7 - 1, + 65536 * 9 + 31 + ]; + + const test = (name, sizes) => { + it(name + " should generate a hash from binary data", async () => { + const hash = createHash(); + const hashString = createHash(); + const reference = await createReferenceHash(); + for (const size of sizes) { + const bytes = randomBytes(size); + const string = bytes.toString("base64"); + hash.update(bytes); + hashString.update(string, "base64"); + reference.update(bytes); + } + const result = hash.digest("hex"); + expect(result).toMatch(regExp); + const resultFromString = hashString.digest("hex"); + expect(resultFromString).toMatch(regExp); + const expected = reference.digest("hex"); + expect(result).toBe(expected); + expect(resultFromString).toBe(expected); + }); + }; + + test("empty hash", []); + + for (const size of sizes) { + test(`single update ${size} bytes`, [size]); + } + + for (const size1 of sizes) { + for (const size2 of sizes) { + test(`two updates ${size1} + ${size2} bytes`, [size1, size2]); + } + } + test(`many updates 1`, sizes); + test(`many updates 2`, sizes.slice().reverse()); + test(`many updates 3`, sizes.concat(sizes.slice().reverse())); + test(`many updates 4`, sizes.slice().reverse().concat(sizes)); + }); +} diff --git a/test/XxHash64.unittest.js b/test/XxHash64.unittest.js deleted file mode 100644 index 4a75a0dd1..000000000 --- a/test/XxHash64.unittest.js +++ /dev/null @@ -1,69 +0,0 @@ -const createHash = require("../lib/util/hash/xxhash64"); -const { randomBytes } = require("crypto"); -const createReferenceHash = - require("hash-wasm/dist/xxhash64.umd.min.js").createXXHash64; - -describe("xxhash64", () => { - const sizes = [ - 1, - 2, - 3, - 4, - 5, - 7, - 8, - 9, - 16, - 31, - 32, - 33, - 64, - 100, - 1000, - 65536 - 1, - 65536, - 65536 + 1, - 65536 + 31, - 65536 * 5, - 65536 * 7 - 1, - 65536 * 9 + 31 - ]; - - const test = (name, sizes) => { - it(name + " should generate a hash from binary data", async () => { - const hash = createHash(); - const hashString = createHash(); - const reference = (await createReferenceHash()).init(); - for (const size of sizes) { - const bytes = randomBytes(size); - const string = bytes.toString("base64"); - hash.update(bytes); - hashString.update(string, "base64"); - reference.update(bytes); - } - const result = hash.digest("hex"); - expect(result).toMatch(/^[0-9a-f]{16}$/); - const resultFromString = hashString.digest("hex"); - expect(resultFromString).toMatch(/^[0-9a-f]{16}$/); - const expected = reference.digest("hex"); - expect(result).toBe(expected); - expect(resultFromString).toBe(expected); - }); - }; - - test("empty hash", []); - - for (const size of sizes) { - test(`single update ${size} bytes`, [size]); - } - - for (const size1 of sizes) { - for (const size2 of sizes) { - test(`two updates ${size1} + ${size2} bytes`, [size1, size2]); - } - } - test(`many updates 1`, sizes); - test(`many updates 2`, sizes.slice().reverse()); - test(`many updates 3`, sizes.concat(sizes.slice().reverse())); - test(`many updates 4`, sizes.slice().reverse().concat(sizes)); -}); diff --git a/tooling/generate-wasm-code.js b/tooling/generate-wasm-code.js index 71ee020d3..d24a18d45 100644 --- a/tooling/generate-wasm-code.js +++ b/tooling/generate-wasm-code.js @@ -6,7 +6,7 @@ const asc = require("assemblyscript/cli/asc"); // Otherwise it only prints outdated files const doWrite = process.argv.includes("--write"); -const files = ["lib/util/hash/xxhash64.js"]; +const files = ["lib/util/hash/xxhash64.js", "lib/util/hash/md4.js"]; (async () => { await asc.ready; diff --git a/types.d.ts b/types.d.ts index 0d7f8e7b4..1e62fa6c7 100644 --- a/types.d.ts +++ b/types.d.ts @@ -7512,6 +7512,7 @@ declare interface NormalModuleLoaderContext { utils: { absolutify: (context: string, request: string) => string; contextify: (context: string, request: string) => string; + createHash: (algorithm: string) => Hash; }; rootContext: string; fs: InputFileSystem;