add wasm md4 implementation

expose createHash is loader context
pull/14584/head
Tobias Koppers 11 months ago
parent f0298fe46f
commit dcf3b02aaf
  1. 177
      assembly/hash/md4.asm.ts
  2. 2
      assembly/hash/xxhash64.asm.ts
  3. 87
      benchmark/md4.js
  4. 2
      declarations/LoaderContext.d.ts
  5. 3
      lib/NormalModule.js
  6. 12
      lib/util/createHash.js
  7. 20
      lib/util/hash/md4.js
  8. 153
      lib/util/hash/wasm-hash.js
  9. 144
      lib/util/hash/xxhash64.js
  10. 96
      test/WasmHashes.unittest.js
  11. 69
      test/XxHash64.unittest.js
  12. 2
      tooling/generate-wasm-code.js
  13. 1
      types.d.ts

@ -0,0 +1,177 @@
/*
** ********************************************************************
** md4.c -- Implementation of MD4 Message Digest Algorithm **
** Updated: 2/16/90 by Ronald L. Rivest **
** (C) 1990 RSA Data Security, Inc. **
** ********************************************************************
*/
// Ported to assemblyscript by Tobias Koppers
let totalLength: u32;
let A: u32;
let B: u32;
let C: u32;
let D: u32;
function F(x: u32, y: u32, z: u32): u32 {
return z ^ (x & (y ^ z));
}
function G(x: u32, y: u32, z: u32): u32 {
return (x & (y | z)) | (y & z);
}
function H(x: u32, y: u32, z: u32): u32 {
return x ^ y ^ z;
}
function roundF(a: u32, b: u32, c: u32, d: u32, i: u32, s: u32): u32 {
return rotl<u32>(a + F(b, c, d) + load<u32>(i), s);
}
function roundG(a: u32, b: u32, c: u32, d: u32, i: u32, s: u32): u32 {
return rotl<u32>(a + G(b, c, d) + load<u32>(i) + 0x5a827999, s);
}
function roundH(a: u32, b: u32, c: u32, d: u32, i: u32, s: u32): u32 {
return rotl<u32>(a + H(b, c, d) + load<u32>(i) + 0x6ed9eba1, s);
}
export function init(): void {
A = 0x67452301;
B = 0xefcdab89;
C = 0x98badcfe;
D = 0x10325476;
totalLength = 0;
}
function body(size: u32): void {
let _A = A;
let _B = B;
let _C = C;
let _D = D;
for (let i: u32 = 0; i < size; i += 64) {
let a = _A;
let b = _B;
let c = _C;
let d = _D;
// Round F
a = roundF(a, b, c, d, i + 4 * 0, 3);
d = roundF(d, a, b, c, i + 4 * 1, 7);
c = roundF(c, d, a, b, i + 4 * 2, 11);
b = roundF(b, c, d, a, i + 4 * 3, 19);
a = roundF(a, b, c, d, i + 4 * 4, 3);
d = roundF(d, a, b, c, i + 4 * 5, 7);
c = roundF(c, d, a, b, i + 4 * 6, 11);
b = roundF(b, c, d, a, i + 4 * 7, 19);
a = roundF(a, b, c, d, i + 4 * 8, 3);
d = roundF(d, a, b, c, i + 4 * 9, 7);
c = roundF(c, d, a, b, i + 4 * 10, 11);
b = roundF(b, c, d, a, i + 4 * 11, 19);
a = roundF(a, b, c, d, i + 4 * 12, 3);
d = roundF(d, a, b, c, i + 4 * 13, 7);
c = roundF(c, d, a, b, i + 4 * 14, 11);
b = roundF(b, c, d, a, i + 4 * 15, 19);
// Round G
a = roundG(a, b, c, d, i + 4 * 0, 3);
d = roundG(d, a, b, c, i + 4 * 4, 5);
c = roundG(c, d, a, b, i + 4 * 8, 9);
b = roundG(b, c, d, a, i + 4 * 12, 13);
a = roundG(a, b, c, d, i + 4 * 1, 3);
d = roundG(d, a, b, c, i + 4 * 5, 5);
c = roundG(c, d, a, b, i + 4 * 9, 9);
b = roundG(b, c, d, a, i + 4 * 13, 13);
a = roundG(a, b, c, d, i + 4 * 2, 3);
d = roundG(d, a, b, c, i + 4 * 6, 5);
c = roundG(c, d, a, b, i + 4 * 10, 9);
b = roundG(b, c, d, a, i + 4 * 14, 13);
a = roundG(a, b, c, d, i + 4 * 3, 3);
d = roundG(d, a, b, c, i + 4 * 7, 5);
c = roundG(c, d, a, b, i + 4 * 11, 9);
b = roundG(b, c, d, a, i + 4 * 15, 13);
// Round H
a = roundH(a, b, c, d, i + 4 * 0, 3);
d = roundH(d, a, b, c, i + 4 * 8, 9);
c = roundH(c, d, a, b, i + 4 * 4, 11);
b = roundH(b, c, d, a, i + 4 * 12, 15);
a = roundH(a, b, c, d, i + 4 * 2, 3);
d = roundH(d, a, b, c, i + 4 * 10, 9);
c = roundH(c, d, a, b, i + 4 * 6, 11);
b = roundH(b, c, d, a, i + 4 * 14, 15);
a = roundH(a, b, c, d, i + 4 * 1, 3);
d = roundH(d, a, b, c, i + 4 * 9, 9);
c = roundH(c, d, a, b, i + 4 * 5, 11);
b = roundH(b, c, d, a, i + 4 * 13, 15);
a = roundH(a, b, c, d, i + 4 * 3, 3);
d = roundH(d, a, b, c, i + 4 * 11, 9);
c = roundH(c, d, a, b, i + 4 * 7, 11);
b = roundH(b, c, d, a, i + 4 * 15, 15);
_A += a;
_B += b;
_C += c;
_D += d;
}
A = _A;
B = _B;
C = _C;
D = _D;
}
export function update(length: u32): void {
body(length);
totalLength += length;
}
export function final(length: u32): void {
const bits: u64 = u64(totalLength + length) << 3;
const finalLength: u32 = (length + 9 + 63) & ~63;
const bitsPosition = finalLength - 8;
// end
store<u8>(length++, 0x80);
// padding
for (; length & 7 && length < finalLength; length++) store<u8>(length, 0);
for (; length < finalLength; length += 8) store<u64>(length, 0);
// bits
store<u64>(bitsPosition, bits);
body(finalLength);
store<u64>(0, u32ToHex(A));
store<u64>(8, u32ToHex(B));
store<u64>(16, u32ToHex(C));
store<u64>(24, u32ToHex(D));
}
function u32ToHex(x: u64): u64 {
// from https://johnnylee-sde.github.io/Fast-unsigned-integer-to-hex-string/
x = ((x & 0xffff0000) << 16) | (x & 0xffff);
x = ((x & 0x0000ff000000ff00) << 8) | (x & 0x000000ff000000ff);
x = ((x & 0x00f000f000f000f0) >> 4) | ((x & 0x000f000f000f000f) << 8);
const mask = ((x + 0x0606060606060606) >> 4) & 0x0101010101010101;
x |= 0x3030303030303030;
x += 0x27 * mask;
return x;
}

@ -108,8 +108,6 @@ export function final(length: u32): void {
result *= Prime3;
result ^= result >> 32;
store<u64>(0, result);
store<u64>(0, u32ToHex(result >> 32));
store<u64>(8, u32ToHex(result & 0xffffffff));
}

@ -0,0 +1,87 @@
const crypto = require("crypto");
const createHash = require("../lib/util/createHash");
let result;
const measure = (fn, count) => {
const start = process.hrtime.bigint();
for (let i = 0; i < count; i++) result = fn();
return Number(process.hrtime.bigint() - start);
};
const NS_PER_MS = 1000000; // 1ms
const MIN_DURATION = 100 * NS_PER_MS; // 100ms
const MAX_DURATION = 1000 * NS_PER_MS; // 1000ms
const MAX_WARMUP_DURATION = 1 * NS_PER_MS; // 1ms
const format = (fast, slow, fastName, slowName, count) => {
return `${fastName} is ${
Math.round(((slow - fast) * 1000) / slow) / 10
}% faster than ${slowName} (${Math.round(fast / 100 / count) / 10} µs vs ${
Math.round(slow / 100 / count) / 10
} µs, ${count}x)`;
};
const compare = (n1, f1, n2, f2) => {
let count = 1;
while (true) {
const timings = [f1, f2, f1, f2, f1, f2].map(f => measure(f, count));
const t1 = Math.min(timings[0], timings[2], timings[4]);
const t2 = Math.min(timings[1], timings[3], timings[5]);
if (count === 1 && (t1 > MAX_WARMUP_DURATION || t2 > MAX_WARMUP_DURATION)) {
continue;
}
if (
(t1 > MIN_DURATION && t2 > MIN_DURATION) ||
t1 > MAX_DURATION ||
t2 > MAX_DURATION
) {
return t1 > t2
? format(t2, t1, n2, n1, count)
: format(t1, t2, n1, n2, count);
}
count *= 2;
}
};
for (const size of [
1, 2, 4, 8, 10, 20, 40, 60, 80, 100, 200, 1000, 5000, 10000, 20000, 50000,
100000, 200000, 500000
]) {
const longString = require("crypto").randomBytes(size).toString("hex");
const buffer = require("crypto").randomBytes(size * 2);
console.log(
`string ${longString.length} chars: ` +
compare(
"crypto md4",
() => {
const hash = crypto.createHash("md4");
hash.update(longString);
return hash.digest("hex");
},
"wasm md4",
() => {
const hash = createHash("md4");
hash.update(longString);
return hash.digest("hex");
}
)
);
console.log(
`buffer ${buffer.length} bytes: ` +
compare(
"crypto md4",
() => {
const hash = crypto.createHash("md4");
hash.update(buffer);
return hash.digest("hex");
},
"wasm md4",
() => {
const hash = createHash("md4");
hash.update(buffer);
return hash.digest("hex");
}
)
);
}

@ -5,6 +5,7 @@ import type { ResolveOptionsWithDependencyType } from "../lib/ResolverFactory";
import type Compilation from "../lib/Compilation";
import type Compiler from "../lib/Compiler";
import type NormalModule from "../lib/NormalModule";
import type Hash from "../lib/util/Hash";
import type { InputFileSystem } from "../lib/util/fs";
import type { Logger } from "../lib/logging/Logger";
import type {
@ -39,6 +40,7 @@ export interface NormalModuleLoaderContext<OptionsType> {
utils: {
absolutify: (context: string, request: string) => string;
contextify: (context: string, request: string) => string;
createHash: (algorithm: string) => Hash;
};
rootContext: string;
fs: InputFileSystem;

@ -538,7 +538,8 @@ class NormalModule extends Module {
return context === this.context
? getContextifyInContext()(request)
: getContextify()(context, request);
}
},
createHash
};
const loaderContext = {
version: 2,

@ -126,6 +126,7 @@ class DebugHash extends Hash {
let crypto = undefined;
let createXXHash64 = undefined;
let createMd4 = undefined;
let BatchedHash = undefined;
/**
@ -149,6 +150,17 @@ module.exports = algorithm => {
}
}
return new BatchedHash(createXXHash64());
case "md4":
if (createMd4 === undefined) {
createMd4 = require("./hash/md4");
if (BatchedHash === undefined) {
BatchedHash = require("./hash/BatchedHash");
}
}
return new BatchedHash(createMd4());
case "native-md4":
if (crypto === undefined) crypto = require("crypto");
return new BulkUpdateDecorator(() => crypto.createHash("md4"), "md4");
default:
if (crypto === undefined) crypto = require("crypto");
return new BulkUpdateDecorator(

@ -0,0 +1,20 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
const create = require("./wasm-hash");
//#region wasm code: md4 (../../../assembly/hash/md4.asm.ts) --initialMemory 1
const md4 = new WebAssembly.Module(
Buffer.from(
// 2150 bytes
"AGFzbQEAAAABCAJgAX8AYAAAAwUEAQAAAAUDAQABBhoFfwFBAAt/AUEAC38BQQALfwFBAAt/AUEACwciBARpbml0AAAGdXBkYXRlAAIFZmluYWwAAwZtZW1vcnkCAAqFEAQmAEGBxpS6BiQBQYnXtv5+JAJB/rnrxXkkA0H2qMmBASQEQQAkAAvMCgEYfyMBIQojAiEGIwMhByMEIQgDQCAAIAVLBEAgBSgCCCINIAcgBiAFKAIEIgsgCCAHIAUoAgAiDCAKIAggBiAHIAhzcXNqakEDdyIDIAYgB3Nxc2pqQQd3IgEgAyAGc3FzampBC3chAiAFKAIUIg8gASACIAUoAhAiCSADIAEgBSgCDCIOIAYgAyACIAEgA3Nxc2pqQRN3IgQgASACc3FzampBA3ciAyACIARzcXNqakEHdyEBIAUoAiAiEiADIAEgBSgCHCIRIAQgAyAFKAIYIhAgAiAEIAEgAyAEc3FzampBC3ciAiABIANzcXNqakETdyIEIAEgAnNxc2pqQQN3IQMgBSgCLCIVIAQgAyAFKAIoIhQgAiAEIAUoAiQiEyABIAIgAyACIARzcXNqakEHdyIBIAMgBHNxc2pqQQt3IgIgASADc3FzampBE3chBCAPIBAgCSAVIBQgEyAFKAI4IhYgAiAEIAUoAjQiFyABIAIgBSgCMCIYIAMgASAEIAEgAnNxc2pqQQN3IgEgAiAEc3FzampBB3ciAiABIARzcXNqakELdyIDIAkgAiAMIAEgBSgCPCIJIAQgASADIAEgAnNxc2pqQRN3IgEgAiADcnEgAiADcXJqakGZ84nUBWpBA3ciAiABIANycSABIANxcmpqQZnzidQFakEFdyIEIAEgAnJxIAEgAnFyaiASakGZ84nUBWpBCXciAyAPIAQgCyACIBggASADIAIgBHJxIAIgBHFyampBmfOJ1AVqQQ13IgEgAyAEcnEgAyAEcXJqakGZ84nUBWpBA3ciAiABIANycSABIANxcmpqQZnzidQFakEFdyIEIAEgAnJxIAEgAnFyampBmfOJ1AVqQQl3IgMgECAEIAIgFyABIAMgAiAEcnEgAiAEcXJqakGZ84nUBWpBDXciASADIARycSADIARxcmogDWpBmfOJ1AVqQQN3IgIgASADcnEgASADcXJqakGZ84nUBWpBBXciBCABIAJycSABIAJxcmpqQZnzidQFakEJdyIDIBEgBCAOIAIgFiABIAMgAiAEcnEgAiAEcXJqakGZ84nUBWpBDXciASADIARycSADIARxcmpqQZnzidQFakEDdyICIAEgA3JxIAEgA3FyampBmfOJ1AVqQQV3IgQgASACcnEgASACcXJqakGZ84nUBWpBCXciAyAMIAIgAyAJIAEgAyACIARycSACIARxcmpqQZnzidQFakENdyIBcyAEc2pqQaHX5/YGakEDdyICIAQgASACcyADc2ogEmpBodfn9gZqQQl3IgRzIAFzampBodfn9gZqQQt3IgMgAiADIBggASADIARzIAJzampBodfn9gZqQQ93IgFzIARzaiANakGh1+f2BmpBA3ciAiAUIAQgASACcyADc2pqQaHX5/YGakEJdyIEcyABc2pqQaHX5/YGakELdyIDIAsgAiADIBYgASADIARzIAJzampBodfn9gZqQQ93IgFzIARzampBodfn9gZqQQN3IgIgEyAEIAEgAnMgA3NqakGh1+f2BmpBCXciBHMgAXNqakGh1+f2BmpBC3chAyAKIA4gAiADIBcgASADIARzIAJzampBodfn9gZqQQ93IgFzIARzampBodfn9gZqQQN3IgJqIQogBiAJIAEgESADIAIgFSAEIAEgAnMgA3NqakGh1+f2BmpBCXciBHMgAXNqakGh1+f2BmpBC3ciAyAEcyACc2pqQaHX5/YGakEPd2ohBiADIAdqIQcgBCAIaiEIIAVBQGshBQwBCwsgCiQBIAYkAiAHJAMgCCQECw0AIAAQASMAIABqJAAL/wQCA38BfiMAIABqrUIDhiEEIABByABqQUBxIgJBCGshAyAAIgFBAWohACABQYABOgAAA0AgACACSUEAIABBB3EbBEAgAEEAOgAAIABBAWohAAwBCwsDQCAAIAJJBEAgAEIANwMAIABBCGohAAwBCwsgAyAENwMAIAIQAUEAIwGtIgRC//8DgyAEQoCA/P8Pg0IQhoQiBEL/gYCA8B+DIARCgP6DgIDgP4NCCIaEIgRCj4C8gPCBwAeDQgiGIARC8IHAh4CegPgAg0IEiIQiBEKGjJiw4MCBgwZ8QgSIQoGChIiQoMCAAYNCJ34gBEKw4MCBg4aMmDCEfDcDAEEIIwKtIgRC//8DgyAEQoCA/P8Pg0IQhoQiBEL/gYCA8B+DIARCgP6DgIDgP4NCCIaEIgRCj4C8gPCBwAeDQgiGIARC8IHAh4CegPgAg0IEiIQiBEKGjJiw4MCBgwZ8QgSIQoGChIiQoMCAAYNCJ34gBEKw4MCBg4aMmDCEfDcDAEEQIwOtIgRC//8DgyAEQoCA/P8Pg0IQhoQiBEL/gYCA8B+DIARCgP6DgIDgP4NCCIaEIgRCj4C8gPCBwAeDQgiGIARC8IHAh4CegPgAg0IEiIQiBEKGjJiw4MCBgwZ8QgSIQoGChIiQoMCAAYNCJ34gBEKw4MCBg4aMmDCEfDcDAEEYIwStIgRC//8DgyAEQoCA/P8Pg0IQhoQiBEL/gYCA8B+DIARCgP6DgIDgP4NCCIaEIgRCj4C8gPCBwAeDQgiGIARC8IHAh4CegPgAg0IEiIQiBEKGjJiw4MCBgwZ8QgSIQoGChIiQoMCAAYNCJ34gBEKw4MCBg4aMmDCEfDcDAAs=",
"base64"
)
);
//#endregion
module.exports = create.bind(null, md4, [], 64, 32);

@ -0,0 +1,153 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
class WasmHash {
/**
* @param {WebAssembly.Instance} instance wasm instance
* @param {WebAssembly.Instance[]} instancesPool pool of instances
* @param {number} chunkSize size of data chunks passed to wasm
* @param {number} digestSize size of digest returned by wasm
*/
constructor(instance, instancesPool, chunkSize, digestSize) {
const exports = /** @type {any} */ (instance.exports);
exports.init();
this.exports = exports;
this.mem = Buffer.from(exports.memory.buffer, 0, 65536);
this.buffered = 0;
this.instancesPool = instancesPool;
this.chunkSize = chunkSize;
this.digestSize = digestSize;
}
reset() {
this.buffered = 0;
this.exports.init();
}
/**
* @param {Buffer | string} data data
* @param {BufferEncoding=} encoding encoding
* @returns {this} itself
*/
update(data, encoding) {
if (typeof data === "string") {
if (data.length < 21845) {
this._updateWithShortString(data, encoding);
return this;
} else {
data = Buffer.from(data, encoding);
}
}
this._updateWithBuffer(data);
return this;
}
/**
* @param {string} data data
* @param {BufferEncoding=} encoding encoding
* @returns {void}
*/
_updateWithShortString(data, encoding) {
const { exports, buffered, mem, chunkSize } = this;
let endPos;
if (data.length < 70) {
if (!encoding || encoding === "utf-8" || encoding === "utf8") {
endPos = buffered;
for (let i = 0; i < data.length; i++) {
const cc = data.charCodeAt(i);
if (cc < 0x80) mem[endPos++] = cc;
else if (cc < 0x800) {
mem[endPos] = (cc >> 6) | 0xc0;
mem[endPos + 1] = (cc & 0x3f) | 0x80;
endPos += 2;
} else {
// bail-out for weird chars
endPos += mem.write(data.slice(endPos), endPos, encoding);
break;
}
}
} else if (encoding === "latin1") {
endPos = buffered;
for (let i = 0; i < data.length; i++) {
const cc = data.charCodeAt(i);
mem[endPos++] = cc;
}
} else {
endPos = buffered + mem.write(data, buffered, encoding);
}
} else {
endPos = buffered + mem.write(data, buffered, encoding);
}
if (endPos < chunkSize) {
this.buffered = endPos;
} else {
const l = endPos & ~(this.chunkSize - 1);
exports.update(l);
const newBuffered = endPos - l;
this.buffered = newBuffered;
if (newBuffered > 0) mem.copyWithin(0, l, endPos);
}
}
/**
* @param {Buffer} data data
* @returns {void}
*/
_updateWithBuffer(data) {
const { exports, buffered, mem } = this;
const length = data.length;
if (buffered + length < this.chunkSize) {
data.copy(mem, buffered, 0, length);
this.buffered += length;
} else {
const l = (buffered + length) & ~(this.chunkSize - 1);
if (l > 65536) {
let i = 65536 - buffered;
data.copy(mem, buffered, 0, i);
exports.update(65536);
const stop = l - buffered - 65536;
while (i < stop) {
data.copy(mem, 0, i, i + 65536);
exports.update(65536);
i += 65536;
}
data.copy(mem, 0, i, l - buffered);
exports.update(l - buffered - i);
} else {
data.copy(mem, buffered, 0, l - buffered);
exports.update(l);
}
const newBuffered = length + buffered - l;
this.buffered = newBuffered;
if (newBuffered > 0) data.copy(mem, 0, length - newBuffered, length);
}
}
digest(type) {
const { exports, buffered, mem, digestSize } = this;
exports.final(buffered);
this.instancesPool.push(this);
return mem.toString("latin1", 0, digestSize);
}
}
const create = (wasmModule, instancesPool, chunkSize, digestSize) => {
if (instancesPool.length > 0) {
const old = instancesPool.pop();
old.reset();
return old;
} else {
return new WasmHash(
new WebAssembly.Instance(wasmModule),
instancesPool,
chunkSize,
digestSize
);
}
};
module.exports = create;

@ -5,150 +5,16 @@
"use strict";
const create = require("./wasm-hash");
//#region wasm code: xxhash64 (../../../assembly/hash/xxhash64.asm.ts) --initialMemory 1
const xxhash64 = new WebAssembly.Module(
Buffer.from(
// 1180 bytes
"AGFzbQEAAAABCAJgAX8AYAAAAwQDAQAABQMBAAEGGgV+AUIAC34BQgALfgFCAAt+AUIAC34BQgALByIEBGluaXQAAAZ1cGRhdGUAAQVmaW5hbAACBm1lbW9yeQIACrwIAzAAQtbrgu7q/Yn14AAkAELP1tO+0ser2UIkAUIAJAJC+erQ0OfJoeThACQDQgAkBAvUAQIBfwR+IABFBEAPCyMEIACtfCQEIwAhAiMBIQMjAiEEIwMhBQNAIAIgASkDAELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiECIAMgASkDCELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEDIAQgASkDEELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEEIAUgASkDGELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEFIAAgAUEgaiIBSw0ACyACJAAgAyQBIAQkAiAFJAMLsgYCAX8EfiMEQgBSBH4jACICQgGJIwEiA0IHiXwjAiIEQgyJfCMDIgVCEol8IAJCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0gA0LP1tO+0ser2UJ+Qh+JQoeVr6+Ytt6bnn9+hUKHla+vmLbem55/fkKdo7Xqg7GNivoAfSAEQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IAVCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0FQsXP2bLx5brqJwsjBCAArXx8IQIDQCABQQhqIABNBEAgAiABKQMAQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQhuJQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IQIgAUEIaiEBDAELCyABQQRqIABNBEACfyACIAE1AgBCh5Wvr5i23puef36FQheJQs/W077Sx6vZQn5C+fPd8Zn2masWfCECIAFBBGoLIQELA0AgACABRwRAIAIgATEAAELFz9my8eW66id+hUILiUKHla+vmLbem55/fiECIAFBAWohAQwBCwtBACACIAJCIYiFQs/W077Sx6vZQn4iAiACQh2IhUL5893xmfaZqxZ+IgIgAkIgiIUiAjcDAEEAIAJCIIgiA0L//wODQiCGIANCgID8/w+DQhCIhCIDQv+BgIDwH4NCEIYgA0KA/oOAgOA/g0IIiIQiA0KPgLyA8IHAB4NCCIYgA0LwgcCHgJ6A+ACDQgSIhCIDQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiADQrDgwIGDhoyYMIR8NwMAQQggAkL/////D4MiAkL//wODQiCGIAJCgID8/w+DQhCIhCICQv+BgIDwH4NCEIYgAkKA/oOAgOA/g0IIiIQiAkKPgLyA8IHAB4NCCIYgAkLwgcCHgJ6A+ACDQgSIhCICQoaMmLDgwIGDBnxCBIhCgYKEiJCgwIABg0InfiACQrDgwIGDhoyYMIR8NwMACw==",
// 1173 bytes
"AGFzbQEAAAABCAJgAX8AYAAAAwQDAQAABQMBAAEGGgV+AUIAC34BQgALfgFCAAt+AUIAC34BQgALByIEBGluaXQAAAZ1cGRhdGUAAQVmaW5hbAACBm1lbW9yeQIACrUIAzAAQtbrgu7q/Yn14AAkAELP1tO+0ser2UIkAUIAJAJC+erQ0OfJoeThACQDQgAkBAvUAQIBfwR+IABFBEAPCyMEIACtfCQEIwAhAiMBIQMjAiEEIwMhBQNAIAIgASkDAELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiECIAMgASkDCELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEDIAQgASkDEELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEEIAUgASkDGELP1tO+0ser2UJ+fEIfiUKHla+vmLbem55/fiEFIAAgAUEgaiIBSw0ACyACJAAgAyQBIAQkAiAFJAMLqwYCAX8EfiMEQgBSBH4jACICQgGJIwEiA0IHiXwjAiIEQgyJfCMDIgVCEol8IAJCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0gA0LP1tO+0ser2UJ+Qh+JQoeVr6+Ytt6bnn9+hUKHla+vmLbem55/fkKdo7Xqg7GNivoAfSAEQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IAVCz9bTvtLHq9lCfkIfiUKHla+vmLbem55/foVCh5Wvr5i23puef35CnaO16oOxjYr6AH0FQsXP2bLx5brqJwsjBCAArXx8IQIDQCABQQhqIABNBEAgAiABKQMAQs/W077Sx6vZQn5CH4lCh5Wvr5i23puef36FQhuJQoeVr6+Ytt6bnn9+Qp2jteqDsY2K+gB9IQIgAUEIaiEBDAELCyABQQRqIABNBEACfyACIAE1AgBCh5Wvr5i23puef36FQheJQs/W077Sx6vZQn5C+fPd8Zn2masWfCECIAFBBGoLIQELA0AgACABRwRAIAIgATEAAELFz9my8eW66id+hUILiUKHla+vmLbem55/fiECIAFBAWohAQwBCwtBACACIAJCIYiFQs/W077Sx6vZQn4iAiACQh2IhUL5893xmfaZqxZ+IgIgAkIgiIUiAkIgiCIDQv//A4NCIIYgA0KAgPz/D4NCEIiEIgNC/4GAgPAfg0IQhiADQoD+g4CA4D+DQgiIhCIDQo+AvIDwgcAHg0IIhiADQvCBwIeAnoD4AINCBIiEIgNChoyYsODAgYMGfEIEiEKBgoSIkKDAgAGDQid+IANCsODAgYOGjJgwhHw3AwBBCCACQv////8PgyICQv//A4NCIIYgAkKAgPz/D4NCEIiEIgJC/4GAgPAfg0IQhiACQoD+g4CA4D+DQgiIhCICQo+AvIDwgcAHg0IIhiACQvCBwIeAnoD4AINCBIiEIgJChoyYsODAgYMGfEIEiEKBgoSIkKDAgAGDQid+IAJCsODAgYOGjJgwhHw3AwAL",
"base64"
)
);
//#endregion
class XxHash64 {
/**
* @param {WebAssembly.Instance} instance wasm instance
*/
constructor(instance) {
const exports = /** @type {any} */ (instance.exports);
exports.init();
this.exports = exports;
this.mem = Buffer.from(exports.memory.buffer, 0, 65536);
this.buffered = 0;
}
reset() {
this.buffered = 0;
this.exports.init();
}
/**
* @param {Buffer | string} data data
* @param {BufferEncoding=} encoding encoding
* @returns {this} itself
*/
update(data, encoding) {
if (typeof data === "string") {
if (data.length < 21845) {
this._updateWithShortString(data, encoding);
return this;
} else {
data = Buffer.from(data, encoding);
}
}
this._updateWithBuffer(data);
return this;
}
/**
* @param {string} data data
* @param {BufferEncoding=} encoding encoding
* @returns {void}
*/
_updateWithShortString(data, encoding) {
const { exports, buffered, mem } = this;
let endPos;
if (data.length < 70) {
if (!encoding || encoding === "utf-8" || encoding === "utf8") {
endPos = buffered;
for (let i = 0; i < data.length; i++) {
const cc = data.charCodeAt(i);
if (cc < 0x80) mem[endPos++] = cc;
else if (cc < 0x800) {
mem[endPos] = (cc >> 6) | 0xc0;
mem[endPos + 1] = (cc & 0x3f) | 0x80;
endPos += 2;
} else {
// bail-out for weird chars
endPos += mem.write(data.slice(endPos), endPos, encoding);
break;
}
}
} else if (encoding === "latin1") {
endPos = buffered;
for (let i = 0; i < data.length; i++) {
const cc = data.charCodeAt(i);
mem[endPos++] = cc;
}
} else {
endPos = buffered + mem.write(data, buffered, encoding);
}
} else {
endPos = buffered + mem.write(data, buffered, encoding);
}
if (endPos < 32) {
this.buffered = endPos;
} else {
const l = (endPos >> 5) << 5;
exports.update(l);
const newBuffered = endPos - l;
this.buffered = newBuffered;
if (newBuffered > 0) mem.copyWithin(0, l, endPos);
}
}
/**
* @param {Buffer} data data
* @returns {void}
*/
_updateWithBuffer(data) {
const { exports, buffered, mem } = this;
const length = data.length;
if (buffered + length < 32) {
data.copy(mem, buffered, 0, length);
this.buffered += length;
} else {
const l = ((buffered + length) >> 5) << 5;
if (l > 65536) {
let i = 65536 - buffered;
data.copy(mem, buffered, 0, i);
exports.update(65536);
const stop = l - buffered - 65536;
while (i < stop) {
data.copy(mem, 0, i, i + 65536);
exports.update(65536);
i += 65536;
}
data.copy(mem, 0, i, l - buffered);
exports.update(l - buffered - i);
} else {
data.copy(mem, buffered, 0, l - buffered);
exports.update(l);
}
const newBuffered = length + buffered - l;
this.buffered = newBuffered;
if (newBuffered > 0) data.copy(mem, 0, length - newBuffered, length);
}
}
digest(type) {
const { exports, buffered, mem } = this;
exports.final(buffered);
instancesPool.push(this);
return mem.toString("latin1", 0, 16);
}
}
const instancesPool = [];
const create = () => {
if (instancesPool.length > 0) {
const old = instancesPool.pop();
old.reset();
return old;
} else {
return new XxHash64(new WebAssembly.Instance(xxhash64));
}
};
module.exports = create;
module.exports = create.bind(null, xxhash64, [], 32, 16);

@ -0,0 +1,96 @@
const { randomBytes, createHash } = require("crypto");
const wasmHashes = {
xxhash64: () => {
const createHash = require("../lib/util/hash/xxhash64");
const createReferenceHash =
require("hash-wasm/dist/xxhash64.umd.min.js").createXXHash64;
return {
createHash,
createReferenceHash: async () => (await createReferenceHash()).init(),
regExp: /^[0-9a-f]{16}$/
};
},
md4: () => {
const createMd4Hash = require("../lib/util/hash/md4");
return {
createHash: createMd4Hash,
createReferenceHash: async () => createHash("md4"),
regExp: /^[0-9a-f]{32}$/
};
}
};
for (const name of Object.keys(wasmHashes)) {
const { createHash, createReferenceHash, regExp } = wasmHashes[name]();
describe(name, () => {
const sizes = [
1,
2,
3,
4,
5,
7,
8,
9,
16,
31,
32,
33,
64 - 10,
64 - 9,
64 - 8,
63,
64,
65,
100,
1000,
65536 - 1,
65536,
65536 + 1,
65536 + 31,
65536 * 5,
65536 * 7 - 1,
65536 * 9 + 31
];
const test = (name, sizes) => {
it(name + " should generate a hash from binary data", async () => {
const hash = createHash();
const hashString = createHash();
const reference = await createReferenceHash();
for (const size of sizes) {
const bytes = randomBytes(size);
const string = bytes.toString("base64");
hash.update(bytes);
hashString.update(string, "base64");
reference.update(bytes);
}
const result = hash.digest("hex");
expect(result).toMatch(regExp);
const resultFromString = hashString.digest("hex");
expect(resultFromString).toMatch(regExp);
const expected = reference.digest("hex");
expect(result).toBe(expected);
expect(resultFromString).toBe(expected);
});
};
test("empty hash", []);
for (const size of sizes) {
test(`single update ${size} bytes`, [size]);
}
for (const size1 of sizes) {
for (const size2 of sizes) {
test(`two updates ${size1} + ${size2} bytes`, [size1, size2]);
}
}
test(`many updates 1`, sizes);
test(`many updates 2`, sizes.slice().reverse());
test(`many updates 3`, sizes.concat(sizes.slice().reverse()));
test(`many updates 4`, sizes.slice().reverse().concat(sizes));
});
}

@ -1,69 +0,0 @@
const createHash = require("../lib/util/hash/xxhash64");
const { randomBytes } = require("crypto");
const createReferenceHash =
require("hash-wasm/dist/xxhash64.umd.min.js").createXXHash64;
describe("xxhash64", () => {
const sizes = [
1,
2,
3,
4,
5,
7,
8,
9,
16,
31,
32,
33,
64,
100,
1000,
65536 - 1,
65536,
65536 + 1,
65536 + 31,
65536 * 5,
65536 * 7 - 1,
65536 * 9 + 31
];
const test = (name, sizes) => {
it(name + " should generate a hash from binary data", async () => {
const hash = createHash();
const hashString = createHash();
const reference = (await createReferenceHash()).init();
for (const size of sizes) {
const bytes = randomBytes(size);
const string = bytes.toString("base64");
hash.update(bytes);
hashString.update(string, "base64");
reference.update(bytes);
}
const result = hash.digest("hex");
expect(result).toMatch(/^[0-9a-f]{16}$/);
const resultFromString = hashString.digest("hex");
expect(resultFromString).toMatch(/^[0-9a-f]{16}$/);
const expected = reference.digest("hex");
expect(result).toBe(expected);
expect(resultFromString).toBe(expected);
});
};
test("empty hash", []);
for (const size of sizes) {
test(`single update ${size} bytes`, [size]);
}
for (const size1 of sizes) {
for (const size2 of sizes) {
test(`two updates ${size1} + ${size2} bytes`, [size1, size2]);
}
}
test(`many updates 1`, sizes);
test(`many updates 2`, sizes.slice().reverse());
test(`many updates 3`, sizes.concat(sizes.slice().reverse()));
test(`many updates 4`, sizes.slice().reverse().concat(sizes));
});

@ -6,7 +6,7 @@ const asc = require("assemblyscript/cli/asc");
// Otherwise it only prints outdated files
const doWrite = process.argv.includes("--write");
const files = ["lib/util/hash/xxhash64.js"];
const files = ["lib/util/hash/xxhash64.js", "lib/util/hash/md4.js"];
(async () => {
await asc.ready;

1
types.d.ts vendored

@ -7512,6 +7512,7 @@ declare interface NormalModuleLoaderContext<OptionsType> {
utils: {
absolutify: (context: string, request: string) => string;
contextify: (context: string, request: string) => string;
createHash: (algorithm: string) => Hash;
};
rootContext: string;
fs: InputFileSystem;

Loading…
Cancel
Save