add support for tree-shaking JSON modules

This commit is contained in:
Tobias Koppers 2019-10-30 23:24:13 +01:00
parent 2ee42bde92
commit 14ee25cd0a
15 changed files with 438 additions and 127 deletions

View File

@ -36,6 +36,7 @@
* @typedef {Object} ExportSpec
* @property {string} name the name of the export
* @property {boolean=} canMangle can the export be renamed (defaults to true)
* @property {(string | ExportSpec)[]=} exports nested exports
* @property {Module=} from when reexported: from which module
* @property {string[] | null=} export when reexported: from which export
*/

View File

@ -115,43 +115,53 @@ class FlagDependencyExportsPlugin {
}
} else if (Array.isArray(exports)) {
// merge in new exports
for (const exportNameOrSpec of exports) {
if (typeof exportNameOrSpec === "string") {
const exportInfo = exportsInfo.getExportInfo(
exportNameOrSpec
);
if (exportInfo.provided === false) {
exportInfo.provided = true;
changed = true;
}
} else {
const exportInfo = exportsInfo.getExportInfo(
exportNameOrSpec.name
);
if (exportInfo.provided === false) {
exportInfo.provided = true;
changed = true;
}
if (exportNameOrSpec.canMangle === false) {
if (exportInfo.canMangleProvide !== false) {
exportInfo.canMangleProvide = false;
const mergeExports = (exportsInfo, exports) => {
for (const exportNameOrSpec of exports) {
if (typeof exportNameOrSpec === "string") {
const exportInfo = exportsInfo.getExportInfo(
exportNameOrSpec
);
if (exportInfo.provided === false) {
exportInfo.provided = true;
changed = true;
}
}
if (exportNameOrSpec.from) {
const fromExportsInfo = moduleGraph.getExportsInfo(
exportNameOrSpec.from
} else {
const exportInfo = exportsInfo.getExportInfo(
exportNameOrSpec.name
);
const nestedExportsInfo = fromExportsInfo.getNestedExportsInfo(
exportNameOrSpec.export
);
if (!exportInfo.exportsInfo && nestedExportsInfo) {
exportInfo.exportsInfo = nestedExportsInfo;
if (exportInfo.provided === false) {
exportInfo.provided = true;
changed = true;
}
if (exportNameOrSpec.canMangle === false) {
if (exportInfo.canMangleProvide !== false) {
exportInfo.canMangleProvide = false;
changed = true;
}
}
if (exportNameOrSpec.exports) {
const nestedExportsInfo = exportInfo.createNestedExportsInfo();
mergeExports(
nestedExportsInfo,
exportNameOrSpec.exports
);
}
if (exportNameOrSpec.from) {
const fromExportsInfo = moduleGraph.getExportsInfo(
exportNameOrSpec.from
);
const nestedExportsInfo = fromExportsInfo.getNestedExportsInfo(
exportNameOrSpec.export
);
if (!exportInfo.exportsInfo && nestedExportsInfo) {
exportInfo.exportsInfo = nestedExportsInfo;
changed = true;
}
}
}
}
}
};
mergeExports(exportsInfo, exports);
}
// store dependencies
if (exportDeps) {

View File

@ -42,41 +42,28 @@ class FlagDependencyUsagePlugin {
const processModule = (module, usedExports) => {
const exportsInfo = moduleGraph.getExportsInfo(module);
if (usedExports.length > 0) {
for (const usedExport of usedExports) {
for (let usedExport of usedExports) {
if (
module.buildMeta.exportsType === "named" &&
usedExport[0] === "default"
) {
usedExport = usedExport.slice(1);
}
if (usedExport.length === 0) {
if (exportsInfo.setUsedInUnknownWay()) {
queue.enqueue(module);
}
} else {
if (
usedExport[0] === "default" &&
module.buildMeta.exportsType === "named"
) {
if (exportsInfo.setUsedAsNamedExportType()) {
queue.enqueue(module);
}
} else {
let currentExportsInfo = exportsInfo;
for (let i = 0; i < usedExport.length; i++) {
const exportName = usedExport[i];
const exportInfo = currentExportsInfo.getExportInfo(
exportName
);
const lastOne = i === usedExport.length - 1;
const nestedInfo = exportInfo.exportsInfo;
if (!nestedInfo || lastOne) {
if (exportInfo.used !== UsageState.Used) {
exportInfo.used = UsageState.Used;
const currentModule =
currentExportsInfo === exportsInfo
? module
: exportInfoToModuleMap.get(currentExportsInfo);
if (currentModule) {
queue.enqueue(currentModule);
}
}
break;
} else {
let currentExportsInfo = exportsInfo;
for (let i = 0; i < usedExport.length; i++) {
const exportName = usedExport[i];
const exportInfo = currentExportsInfo.getExportInfo(
exportName
);
const lastOne = i === usedExport.length - 1;
if (!lastOne) {
const nestedInfo = exportInfo.getNestedExportsInfo();
if (nestedInfo) {
if (exportInfo.used === UsageState.Unused) {
exportInfo.used = UsageState.OnlyPropertiesUsed;
const currentModule =
@ -88,8 +75,20 @@ class FlagDependencyUsagePlugin {
}
}
currentExportsInfo = nestedInfo;
continue;
}
}
if (exportInfo.used !== UsageState.Used) {
exportInfo.used = UsageState.Used;
const currentModule =
currentExportsInfo === exportsInfo
? module
: exportInfoToModuleMap.get(currentExportsInfo);
if (currentModule) {
queue.enqueue(currentModule);
}
}
break;
}
}
}

View File

@ -119,6 +119,9 @@ class ExportsInfo {
if (exportInfo.canMangleUse === undefined) {
exportInfo.canMangleUse = true;
}
if (exportInfo.exportsInfoOwned) {
exportInfo.exportsInfo.setHasUseInfo();
}
}
if (this._otherExportsInfo.used === UsageState.NoInfo) {
this._otherExportsInfo.used = UsageState.Unused;
@ -481,6 +484,8 @@ class ExportInfo {
* @type {boolean | undefined}
*/
this.canMangleUse = initFrom ? initFrom.canMangleUse : undefined;
/** @type {boolean} */
this.exportsInfoOwned = false;
/** @type {ExportsInfo=} */
this.exportsInfo = undefined;
}
@ -511,6 +516,18 @@ class ExportInfo {
return this.usedName || this.name || fallbackName;
}
createNestedExportsInfo() {
if (this.exportsInfoOwned) return this.exportsInfo;
this.exportsInfoOwned = true;
this.exportsInfo = new ExportsInfo();
this.exportsInfo.setHasProvideInfo();
return this.exportsInfo;
}
getNestedExportsInfo() {
return this.exportsInfo;
}
getUsedInfo() {
switch (this.used) {
case UsageState.NoInfo:

View File

@ -593,7 +593,7 @@ class RuntimeTemplate {
if (exportsType === "named") {
if (exportName.length > 0 && exportName[0] === "default") {
return `${importVar}${propertyAccess(exportName, 1)}`;
exportName = exportName.slice(1);
} else if (exportName.length === 0) {
return `${importVar}_namespace`;
}
@ -604,13 +604,13 @@ class RuntimeTemplate {
const used = exportsInfo.getUsedName(exportName);
if (!used) {
const comment = Template.toNormalComment(
`unused export ${exportName.join(".")}`
`unused export ${propertyAccess(exportName)}`
);
return `${comment} undefined`;
}
const comment = arrayEquals(used, exportName)
? ""
: Template.toNormalComment(exportName.join(".")) + " ";
: Template.toNormalComment(propertyAccess(exportName)) + " ";
const access = `${importVar}${comment}${propertyAccess(used)}`;
if (isCall && callContext === false) {
if (asiSafe) {

View File

@ -0,0 +1,97 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
const makeSerializable = require("../util/makeSerializable");
const NullDependency = require("./NullDependency");
/** @typedef {import("../ChunkGraph")} ChunkGraph */
/** @typedef {import("../Dependency").ExportSpec} ExportSpec */
/** @typedef {import("../Dependency").ExportsSpec} ExportsSpec */
/** @typedef {import("../ModuleGraph")} ModuleGraph */
/** @typedef {import("../util/Hash")} Hash */
const getExportsFromData = data => {
if (data && typeof data === "object") {
if (Array.isArray(data)) {
return data.map((item, idx) => {
return {
name: `${idx}`,
canMangle: true,
exports: getExportsFromData(item)
};
});
} else {
const exports = [];
for (const key of Object.keys(data)) {
exports.push({
name: key,
canMangle: true,
exports: getExportsFromData(data[key])
});
}
return exports;
}
}
return undefined;
};
class JsonExportsDependency extends NullDependency {
/**
* @param {(string | ExportSpec)[]} exports json exports
*/
constructor(exports) {
super();
this.exports = exports;
}
get type() {
return "json exports";
}
/**
* Returns the exported names
* @param {ModuleGraph} moduleGraph module graph
* @returns {ExportsSpec | undefined} export names
*/
getExports(moduleGraph) {
return {
exports: this.exports,
dependencies: undefined
};
}
/**
* Update the hash
* @param {Hash} hash hash to be updated
* @param {ChunkGraph} chunkGraph chunk graph
* @returns {void}
*/
updateHash(hash, chunkGraph) {
hash.update(this.exports ? JSON.stringify(this.exports) : "undefined");
super.updateHash(hash, chunkGraph);
}
serialize(context) {
const { write } = context;
write(this.exports);
super.serialize(context);
}
deserialize(context) {
const { read } = context;
this.exports = read();
super.deserialize(context);
}
}
makeSerializable(
JsonExportsDependency,
"webpack/lib/dependencies/JsonExportsDependency"
);
module.exports = JsonExportsDependency;
module.exports.getExportsFromData = getExportsFromData;

View File

@ -12,6 +12,7 @@ const RuntimeGlobals = require("../RuntimeGlobals");
/** @typedef {import("webpack-sources").Source} Source */
/** @typedef {import("../Generator").GenerateContext} GenerateContext */
/** @typedef {import("../ModuleGraph").ExportsInfo} ExportsInfo */
/** @typedef {import("../NormalModule")} NormalModule */
const stringifySafe = data => {
@ -25,6 +26,51 @@ const stringifySafe = data => {
); // invalid in JavaScript but valid JSON
};
/**
* @param {Object} data data (always an object or array)
* @param {ExportsInfo} exportsInfo exports info
* @returns {Object} reduced data
*/
const createObjectForExportsInfo = (data, exportsInfo) => {
if (exportsInfo.otherExportsInfo.used !== UsageState.Unused) return data;
const reducedData = Array.isArray(data) ? [] : {};
for (const exportInfo of exportsInfo.exports) {
if (exportInfo.name in reducedData) return data;
}
for (const key of Object.keys(data)) {
const exportInfo = exportsInfo.getReadOnlyExportInfo(key);
if (exportInfo.used === UsageState.Unused) continue;
let value;
if (
exportInfo.used === UsageState.OnlyPropertiesUsed &&
exportInfo.exportsInfo
) {
value = createObjectForExportsInfo(data[key], exportInfo.exportsInfo);
} else {
value = data[key];
}
const name = exportInfo.getUsedName(key);
reducedData[name] = value;
}
if (Array.isArray(reducedData)) {
let sizeObjectMinusArray = 0;
for (let i = 0; i < reducedData.length; i++) {
if (reducedData[i] === undefined) {
sizeObjectMinusArray -= 2;
} else {
sizeObjectMinusArray += `${i}`.length + 3;
}
}
if (sizeObjectMinusArray < 0) return Object.assign({}, reducedData);
for (let i = 0; i < reducedData.length; i++) {
if (reducedData[i] === undefined) {
reducedData[i] = 0;
}
}
}
return reducedData;
};
const TYPES = new Set(["javascript"]);
class JsonGenerator extends Generator {
@ -62,28 +108,15 @@ class JsonGenerator extends Generator {
);
}
runtimeRequirements.add(RuntimeGlobals.module);
const providedExports = moduleGraph.getProvidedExports(module);
let finalJson;
if (
Array.isArray(providedExports) &&
module.isExportUsed(moduleGraph, "default") === UsageState.Unused
) {
// Only some exports are used: We can optimize here, by only generating a part of the JSON
const reducedJson = {};
for (const exportName of providedExports) {
if (exportName === "default") continue;
const used = module.getUsedName(moduleGraph, exportName);
if (typeof used === "string") {
reducedJson[used] = data[exportName];
}
}
finalJson = reducedJson;
} else {
finalJson = data;
}
const exportsInfo = moduleGraph.getExportsInfo(module);
let finalJson =
typeof data === "object" && data
? createObjectForExportsInfo(data, exportsInfo)
: data;
// Use JSON because JSON.parse() is much faster than JavaScript evaluation
const jsonSource = JSON.stringify(stringifySafe(finalJson));
const jsonExpr = `JSON.parse(${jsonSource})`;
const jsonStr = stringifySafe(finalJson);
const jsonExpr =
jsonStr.length > 20 ? `JSON.parse(${JSON.stringify(jsonStr)})` : jsonStr;
source.add(`${module.moduleArgument}.exports = ${jsonExpr};`);
return source;
}

View File

@ -6,7 +6,7 @@
"use strict";
const parseJson = require("json-parse-better-errors");
const StaticExportsDependency = require("../dependencies/StaticExportsDependency");
const JsonExportsDependency = require("../dependencies/JsonExportsDependency");
/** @typedef {import("../NormalModule").ParserState} ParserState */
@ -20,12 +20,9 @@ class JsonParser {
const data = parseJson(source[0] === "\ufeff" ? source.slice(1) : source);
state.module.buildInfo.jsonData = data;
state.module.buildMeta.exportsType = "named";
if (typeof data === "object" && data) {
state.module.addDependency(
new StaticExportsDependency(Object.keys(data), true)
);
}
state.module.addDependency(new StaticExportsDependency(["default"], true));
state.module.addDependency(
new JsonExportsDependency(JsonExportsDependency.getExportsFromData(data))
);
return state;
}
}

View File

@ -5,6 +5,7 @@
"use strict";
const { UsageState } = require("../ModuleGraph");
const { numberToIdentifier } = require("../Template");
const { assignDeterministicIds } = require("../ids/IdHelpers");
const {
@ -14,14 +15,29 @@ const {
} = require("../util/comparators");
/** @typedef {import("../Compiler")} Compiler */
/** @typedef {import("../ModuleGraph").ExportsInfo} ExportsInfo */
const canMangleSomething = exportsInfo => {
const OBJECT = [];
const ARRAY = [];
/**
* @param {ExportsInfo} exportsInfo exports info
* @param {boolean} canBeArray can be exports info point to an array
* @returns {boolean} mangle is possible
*/
const canMangle = (exportsInfo, canBeArray) => {
if (exportsInfo.otherExportsInfo.used !== UsageState.Unused) return false;
let hasSomethingToMangle = false;
const empty = canBeArray ? ARRAY : OBJECT;
for (const exportInfo of exportsInfo.exports) {
if (exportInfo.name in empty) {
return false;
}
if (exportInfo.canMangle === true) {
return true;
hasSomethingToMangle = true;
}
}
return false;
return hasSomethingToMangle;
};
const comparator = concatComparators(
@ -31,6 +47,53 @@ const comparator = concatComparators(
compareSelect(e => e.name, compareStringsNumeric)
);
/**
* @param {ExportsInfo} exportsInfo exports info
* @param {boolean} canBeArray can be exports info point to an array
* @returns {void}
*/
const mangleExportsInfo = (exportsInfo, canBeArray) => {
if (!canMangle(exportsInfo, canBeArray)) return;
const usedNames = new Set();
const mangleableExports = [];
// Don't rename 1-2 char exports or exports that can't be mangled
for (const exportInfo of exportsInfo.exports) {
const name = exportInfo.name;
if (
exportInfo.canMangle !== true ||
(name.length === 1 && /^[a-zA-Z0-9_$]/.test(name)) ||
(name.length === 2 && /^[a-zA-Z_$][a-zA-Z0-9_$]|^[1-9][0-9]/.test(name))
) {
exportInfo.usedName = name;
usedNames.add(name);
} else {
mangleableExports.push(exportInfo);
}
if (
exportInfo.exportsInfoOwned &&
exportInfo.used === UsageState.OnlyPropertiesUsed
) {
mangleExportsInfo(exportInfo.exportsInfo, true);
}
}
assignDeterministicIds(
mangleableExports,
e => e.name,
comparator,
(e, id) => {
const name = numberToIdentifier(id);
const size = usedNames.size;
usedNames.add(name);
if (size === usedNames.size) return false;
e.usedName = name;
return true;
},
[26, 52],
52,
usedNames.size
);
};
class MangleExportsPlugin {
/**
* @param {Compiler} compiler webpack compiler
@ -44,38 +107,7 @@ class MangleExportsPlugin {
modules => {
for (const module of modules) {
const exportsInfo = moduleGraph.getExportsInfo(module);
if (!canMangleSomething(exportsInfo)) continue;
const usedNames = new Set();
const mangleableExports = [];
// Don't rename single char exports or exports that can't be mangled
for (const exportInfo of exportsInfo.exports) {
const name = exportInfo.name;
if (
exportInfo.canMangle !== true ||
(name.length === 1 || /^a-zA-Z_\$$/.test(name))
) {
exportInfo.usedName = name;
usedNames.add(name);
} else {
mangleableExports.push(exportInfo);
}
}
assignDeterministicIds(
mangleableExports,
e => e.name,
comparator,
(e, id) => {
const name = numberToIdentifier(id);
const size = usedNames.size;
usedNames.add(name);
if (size === usedNames.size) return false;
e.usedName = name;
return true;
},
[26, 52],
52,
usedNames.size
);
mangleExportsInfo(exportsInfo, false);
}
}
);

View File

@ -74,6 +74,8 @@ module.exports = {
require("../dependencies/ImportEagerDependency"),
"dependencies/ImportWeakDependency": () =>
require("../dependencies/ImportWeakDependency"),
"dependencies/JsonExportsDependency": () =>
require("../dependencies/JsonExportsDependency"),
"dependencies/LocalModule": () => require("../dependencies/LocalModule"),
"dependencies/LocalModuleDependency": () =>
require("../dependencies/LocalModuleDependency"),

View File

@ -3701,7 +3701,7 @@ Built at: 1970-04-20 12:42:42
256e72dd8b9a83a6e45b.module.wasm 120 bytes [emitted] [immutable]
325.bundle.js 3.71 KiB [emitted]
526.bundle.js 368 bytes [emitted] [id hint: vendors]
780.bundle.js 495 bytes [emitted]
780.bundle.js 496 bytes [emitted]
99.bundle.js 205 bytes [emitted]
a0e9dd97d7ced35a5b2c.module.wasm 154 bytes [emitted] [immutable]
bundle.js 11 KiB [emitted] [name: main-1df31ce3]

View File

@ -0,0 +1,58 @@
{
"UNUSEDKEY": "UNUSEDVALUE",
"nested": {
"UNUSEDKEY": "UNUSEDVALUE",
"key": "value",
"key2": "value2",
"array": [1, 2, 3],
"array2": [1, 2, 3],
"array3": ["UNUSEDVALUE", "ok", "UNUSEDVALUE"],
"array4": [
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"ok"
],
"array5": [
"ok",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE",
"UNUSEDVALUE"
],
"object": {
"test": "TESTVALUE"
},
"object2": {},
"object3": {},
"number": 42
},
"nestedArray": [
"UNUSEDVALUE",
[
"UNUSEDVALUE",
{
"UNUSEDKEY": "UNUSEDVALUE",
"deep": {
"UNUSEDKEY": "UNUSEDVALUE",
"deep": "ok"
}
},
"UNUSEDVALUE"
],
"UNUSEDVALUE"
]
}

View File

@ -0,0 +1,57 @@
import data, { nestedArray } from "./data";
import packageJson from "../../../../package.json";
it("should have to correct values", () => {
expect(data.nested.key).toBe("value");
});
it("should be able to write properties", () => {
// known key
data.nested.key2 = "new value";
expect(data.nested.key2).toBe("new value");
// unknown key
data.nested.key3 = "value3";
expect(data.nested.key3).toBe("value3");
// array methods and prototype properties
data.nested.array.push(4);
expect(data.nested.array.length).toEqual(4);
// direct and nested access
const a = data.nested.array2;
data.nested.array2[1] = 42;
expect(a[1]).toEqual(42);
expect(a.length).toEqual(3);
// only nested access
expect(data.nested.array3[1]).toBe("ok");
expect(data.nested.array4[10]).toBe("ok");
expect(data.nested.array5[0]).toBe("ok");
// object methods
expect(data.nested.object.hasOwnProperty("test")).toBe(true);
// unknown object property
data.nested.object2.MANGLE_ME = 42;
expect(data.nested.object2.MANGLE_ME).toBe(42);
data.nested.object3.MANGLE_ME = { deep: "deep" };
expect(data.nested.object3.MANGLE_ME.deep).toBe("deep");
// number methods
expect(data.nested.number.toFixed(1)).toBe("42.0");
// nested in array
expect(nestedArray[1][1].deep.deep).toBe("ok");
});
it("should not have unused keys in bundle", () => {
const fs = require("fs");
const content = fs.readFileSync(__filename, "utf-8");
expect(content).toMatch(/\\?"TESTVALUE\\?"/);
expect(content).not.toMatch(/\\?"UNUSEDKEY\\?"/);
expect(content).not.toMatch(/\\?"UNUSEDVALUE\\?"/);
expect(content).not.toMatch(/\\?"nested\\?"/);
expect(content).not.toMatch(/\.MANGLE_ME(\.deep)?(\s*=|\))/);
expect(content).not.toMatch(/\\?"dependencies\\?"/);
expect(content).not.toMatch(/\\?"scripts\\?"/);
});
it("should tree-shake package.json too", () => {
expect(packageJson.name).toBe("webpack");
expect(packageJson.repository.url).toBe(
"https://github.com/webpack/webpack.git"
);
});

View File

@ -0,0 +1 @@
module.exports = [[/Can't import the named export/]];

View File

@ -0,0 +1,7 @@
module.exports = {
mode: "production",
node: {
__dirname: false,
__filename: false
}
};