Browse Source

added experimental deduplication support #47

0.10
Tobias Koppers 9 years ago
parent
commit
c30f16f439
  1. 2
      bin/config-optimist.js
  2. 5
      bin/convert-argv.js
  3. 2
      lib/FunctionModuleTemplate.js
  4. 5
      lib/JsonpMainTemplate.js
  5. 18
      lib/MainTemplate.js
  6. 4
      lib/Module.js
  7. 82
      lib/NormalModule.js
  8. 3
      lib/Stats.js
  9. 4
      lib/WebpackOptionsApply.js
  10. 4
      lib/dependencies/ModuleDependencyTemplateAsId.js
  11. 4
      lib/dependencies/ModuleDependencyTemplateAsRequireId.js
  12. 27
      lib/dependencies/TemplateArgumentDependency.js
  13. 5
      lib/node/NodeMainTemplate.js
  14. 190
      lib/optimize/DedupePlugin.js
  15. 5
      lib/webworker/WebWorkerMainTemplate.js
  16. 2
      package.json
  17. 4
      test/browsertest/build.js
  18. 1
      test/browsertest/lib/dedupe1/dupdep.js
  19. 1
      test/browsertest/lib/dedupe1/index.js
  20. 1
      test/browsertest/lib/dedupe2/dupdep.js
  21. 1
      test/browsertest/lib/dedupe2/index.js
  22. 9
      test/browsertest/lib/index.web.js

2
bin/config-optimist.js

@ -61,6 +61,8 @@ module.exports = function(optimist) {
.boolean("optimize-occurence-order").describe("optimize-occurence-order")
.boolean("optimize-dedupe").describe("optimize-dedupe")
.string("prefetch").describe("prefetch")
.string("provide").describe("provide")

5
bin/convert-argv.js

@ -267,6 +267,11 @@ module.exports = function(optimist, argv, convertOptions) {
options.optimize.occurenceOrder = true;
});
ifBooleanArg("optimize-dedupe", function() {
ensureObject(options, "optimize");
options.optimize.dedupe = true;
});
ifArg("prefetch", function(request) {
ensureArray(options, "prefetch");
options.prefetch.push(request);

2
lib/FunctionModuleTemplate.js

@ -19,7 +19,7 @@ FunctionModuleTemplate.prototype.render = function(module, dependencyTemplates)
source.add(" !*** " + req.replace(/\*\//g, "*_/") + " ***!\n");
source.add(" \\****" + req.replace(/./g, "*") + "****/\n");
}
source.add("/***/ function(module, exports, require) {\n\n");
source.add("/***/ function(" + ["module", "exports", "require"].concat(module.arguments || []).join(", ") + ") {\n\n");
source.add(new PrefixSource("\t", module.source(dependencyTemplates, this.outputOptions, this.requestShortener)));
source.add("\n\n/***/ }");
return source;

5
lib/JsonpMainTemplate.js

@ -84,8 +84,9 @@ JsonpMainTemplate.prototype.renderInit = function(hash, chunk) {
"installedChunks[chunkId] = 0;"
]),
"}",
"for(moduleId in moreModules)",
this.indent("modules[moduleId] = moreModules[moduleId];"),
"for(moduleId in moreModules) {",
this.indent(this.renderAddModule(hash, chunk, "moduleId", "moreModules[moduleId]")),
"}",
"while(callbacks.length)",
this.indent("callbacks.shift().call(null, require);"),
]),

18
lib/MainTemplate.js

@ -33,7 +33,16 @@ MainTemplate.prototype.render = function(hash, chunk, moduleTemplate, dependency
source.add(new PrefixSource("/******/ \t", new OriginalSource(this.asString(buf), "webpackBootstrap " + hash)));
source.add("\n/******/ })\n");
source.add("/************************************************************************/\n");
source.add("/******/ ({\n");
source.add("/******/ (");
source.add(this.renderModules(hash, chunk, moduleTemplate, dependencyTemplates));
source.add(")");
chunk.rendered = true;
return source;
};
MainTemplate.prototype.renderModules = function renderModules(hash, chunk, moduleTemplate, dependencyTemplates) {
var source = new ConcatSource();
source.add("{\n");
source.add(this.asString(this.renderInitModules(hash, chunk, moduleTemplate, dependencyTemplates)));
source.add("\n");
chunk.modules.forEach(function(module, idx) {
@ -41,8 +50,7 @@ MainTemplate.prototype.render = function(hash, chunk, moduleTemplate, dependency
source.add("\n/***/ " + module.id + ":\n");
source.add(moduleTemplate.render(module, dependencyTemplates));
});
source.add("\n/******/ })");
chunk.rendered = true;
source.add("\n/******/ }");
return source;
};
@ -138,6 +146,10 @@ MainTemplate.prototype.renderInitModules = function(hash, chunk, moduleTemplate,
];
};
MainTemplate.prototype.renderAddModule = function(hash, chunk, varModuleId, varModule) {
return ["modules[" + varModuleId + "] = " + varModule + ";"]
}
MainTemplate.prototype.updateHash = function(hash) {
hash.update("maintemplate");
hash.update("1");

4
lib/Module.js

@ -23,10 +23,6 @@ module.exports = Module;
Module.prototype = Object.create(DependenciesBlock.prototype);
Module.prototype.separable = function(callback) {
callback(false);
};
Module.prototype.disconnect = function() {
this.reasons.length = 0;
this.lastId = this.id;

82
lib/NormalModule.js

@ -9,6 +9,7 @@ var OriginalSource = require("webpack-core/lib/OriginalSource");
var RawSource = require("webpack-core/lib/RawSource");
var ReplaceSource = require("webpack-core/lib/ReplaceSource");
var ModuleParseError = require("./ModuleParseError");
var TemplateArgumentDependency = require("./dependencies/TemplateArgumentDependency");
var path = require("path");
function NormalModule(request, userRequest, rawRequest, loaders, resource, parser) {
@ -82,7 +83,7 @@ NormalModule.prototype.source = function(dependencyTemplates, outputOptions, req
function doDep(dep) {
var template = dependencyTemplates.get(dep.Class);
if(!template) throw new Error("No template for dependency: " + dep.Class.name);
template.apply(dep, source, outputOptions, requestShortener);
template.apply(dep, source, outputOptions, requestShortener, dependencyTemplates);
}
function doVariable(vars, variable) {
var name = variable.name;
@ -151,3 +152,82 @@ NormalModule.prototype.updateHash = function(hash) {
hash.update("null");
Module.prototype.updateHash.call(this, hash);
};
NormalModule.prototype.getSourceHash = function() {
if(!this._source) return "";
var hash = new (require("crypto").Hash)("md5");
hash.update(this._source.source());
return hash.digest("hex");
};
NormalModule.prototype.getAllModuleDependencies = function() {
var list = [];
function doDep(dep) {
if(dep.module && list.indexOf(dep.module) < 0)
list.push(dep.module);
}
function doVariable(variable) {
variable.dependencies.forEach(doDep);
}
function doBlock(block) {
block.variables.forEach(doVariable);
block.dependencies.forEach(doDep);
block.blocks.forEach(doBlock);
}
doBlock(this);
return list;
};
NormalModule.prototype.createTemplate = function(keepModules) {
var template = new NormalModule("", "", "", [], "", null);
template._source = this._source;
template.built = this.built;
template.templateModules = keepModules;
var args = template.arguments = [];
function doDeps(deps) {
return deps.map(function(dep) {
if(keepModules.indexOf(dep.module) < 0) {
var argName = "__webpack_module_template_argument_" + args.length + "__";
args.push(argName);
return new TemplateArgumentDependency(argName, dep);
} else {
return dep;
}
});
}
function doVariable(variable, newVariable) {
variable.dependencies.forEach(doDep);
}
function doBlock(block, newBlock) {
block.variables.forEach(function(variable) {
var newDependencies = doDeps(variable.dependencies);
newBlock.addVariable(variable.name, variable.expression, newDependencies);
});
newBlock.dependencies = doDeps(block.dependencies);
block.blocks.forEach(function(childBlock) {
var newChildBlock = new AsyncDependenciesBlock(childBlock.name);
newBlock.addBlock(newChildBlock);
doBlock(childBlock, newChildBlock);
});
}
doBlock(this, template);
return template;
};
NormalModule.prototype.getTemplateArguments = function(keepModules) {
var list = [];
function doDep(dep) {
if(dep.module && keepModules.indexOf(dep.module) < 0)
list.push(dep.module);
}
function doVariable(variable) {
variable.dependencies.forEach(doDep);
}
function doBlock(block) {
block.variables.forEach(doVariable);
block.dependencies.forEach(doDep);
block.blocks.forEach(doBlock);
}
doBlock(this);
return list;
};

3
lib/Stats.js

@ -135,6 +135,7 @@ Stats.prototype.toJson = function toJson(options, forToString) {
userRequest: reason.dependency.userRequest
}
var dep = reason.dependency;
if(dep.templateModules) obj.templateModules = dep.templateModules.map(function(module) { return module.id; });
if(dep.loc) obj.loc = dep.loc.start.line + ":" + dep.loc.start.column + "-" +
(dep.loc.start.line != dep.loc.end.line ? dep.loc.end.line + ":" : "") + dep.loc.end.column
return obj;
@ -436,6 +437,7 @@ Stats.jsonToString = function jsonToString(obj, useColors) {
normal(reason.type);
normal(" ");
cyan(reason.userRequest);
if(reason.templateModules) cyan(reason.templateModules.join(" "));
normal(" [");
normal(reason.moduleId);
normal("] ");
@ -469,6 +471,7 @@ Stats.jsonToString = function jsonToString(obj, useColors) {
normal(reason.type);
normal(" ");
cyan(reason.userRequest);
if(reason.templateModules) cyan(reason.templateModules.join(" "));
normal(" [");
normal(reason.moduleId);
normal("] ");

4
lib/WebpackOptionsApply.js

@ -42,6 +42,7 @@ var RemoveParentModulesPlugin = require("./optimize/RemoveParentModulesPlugin");
var RemoveEmptyChunksPlugin = require("./optimize/RemoveEmptyChunksPlugin");
var MergeDuplicateChunksPlugin = require("./optimize/MergeDuplicateChunksPlugin");
var FlagIncludedChunksPlugin = require("./optimize/FlagIncludedChunksPlugin");
var DedupePlugin = require("./optimize/DedupePlugin");
var ModulesInDirectoriesPlugin = require("enhanced-resolve/lib/ModulesInDirectoriesPlugin");
var ModulesInRootPlugin = require("enhanced-resolve/lib/ModulesInRootPlugin");
@ -156,6 +157,9 @@ WebpackOptionsApply.prototype.process = function(options, compiler) {
else if(options.optimize.minimize)
compiler.apply(new UglifyJsPlugin(options.optimize.minimize));
if(options.optimize.dedupe === true)
compiler.apply(new DedupePlugin());
if(options.cache === undefined ? options.watch : options.cache)
compiler.apply(new CachePlugin(typeof options.cache == "object" ? options.cache : null));

4
lib/dependencies/ModuleDependencyTemplateAsId.js

@ -14,3 +14,7 @@ ModuleDependencyTemplateAsId.prototype.apply = function(dep, source, outputOptio
var content = "(function webpackMissingModule() { throw new Error(" + JSON.stringify("Cannot find module \"" + dep.request + "\"") + "); }())";
source.replace(dep.range[0], dep.range[1]-1, content);
};
ModuleDependencyTemplateAsId.prototype.applyAsTemplateArgument = function(name, dep, source, outputOptions, requestShortener) {
source.replace(dep.range[0], dep.range[1]-1, name);
};

4
lib/dependencies/ModuleDependencyTemplateAsRequireId.js

@ -14,3 +14,7 @@ ModuleDependencyTemplateAsRequireId.prototype.apply = function(dep, source, outp
var content = "(function webpackMissingModule() { throw new Error(" + JSON.stringify("Cannot find module \"" + dep.request + "\"") + "); }())";
source.replace(dep.range[0], dep.range[1]-1, content);
};
ModuleDependencyTemplateAsRequireId.prototype.applyAsTemplateArgument = function(name, dep, source, outputOptions, requestShortener) {
source.replace(dep.range[0], dep.range[1]-1, "(require(" + name + "))");
};

27
lib/dependencies/TemplateArgumentDependency.js

@ -0,0 +1,27 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
function TemplateArgumentDependency(name, dep) {
this.name = name;
this.Class = TemplateArgumentDependency;
this.dep = dep;
}
module.exports = TemplateArgumentDependency;
TemplateArgumentDependency.prototype.type = "template argument";
TemplateArgumentDependency.prototype.updateHash = function(hash) {
hash.update(this.name);
};
TemplateArgumentDependency.Template = function TemplateArgumentDependencyTemplate() {};
TemplateArgumentDependency.Template.prototype.apply = function(dep, source, outputOptions, requestShortener, dependencyTemplates) {
var d = dep.dep;
var template = dependencyTemplates.get(d.Class);
if(!template) throw new Error("No template for dependency: " + d.Class.name);
if(!template.applyAsTemplateArgument) throw new Error("Template cannot be applied as TemplateArgument: " + d.Class.name);
return template.applyAsTemplateArgument(dep.name, d, source, outputOptions, requestShortener, dependencyTemplates);
};

5
lib/node/NodeMainTemplate.js

@ -39,8 +39,9 @@ NodeMainTemplate.prototype.renderRequireEnsure = function(hash, chunk) {
.replace(Template.REGEXP_NAME, ""))
.replace(Template.REGEXP_ID, "\" + chunkId + \"") + ");",
"var moreModules = chunk.modules, chunkIds = chunk.ids;",
"for(var moduleId in moreModules)",
this.indent("modules[moduleId] = moreModules[moduleId];"),
"for(var moduleId in moreModules) {",
this.indent(this.renderAddModule(hash, chunk, "moduleId", "moreModules[moduleId]")),
"}",
"for(var i = 0; i < chunkIds.length; i++)",
this.indent("installedChunks[chunkIds[i]] = 1;"),
]),

190
lib/optimize/DedupePlugin.js

@ -0,0 +1,190 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
var ConcatSource = require("webpack-core/lib/ConcatSource");
var TemplateArgumentDependency = require("../dependencies/TemplateArgumentDependency");
function DedupePlugin() {
}
module.exports = DedupePlugin;
DedupePlugin.prototype.apply = function(compiler) {
compiler.plugin("compilation", function(compilation) {
compilation.dependencyTemplates.set(TemplateArgumentDependency, new TemplateArgumentDependency.Template());
compilation.plugin("after-optimize-modules", function(modules) {
var modulesByHash = {};
var allDups = [];
modules.forEach(function(module, idx) {
if(!module.getSourceHash || !module.getAllModuleDependencies || !module.createTemplate || !module.getTemplateArguments) return;
var hash = module.getSourceHash();
var dupModule = modulesByHash[hash];
if(dupModule) {
if(dupModule.duplicates) {
dupModule.duplicates.push(module);
module.duplicates = dupModule.duplicates;
} else {
allDups.push(module.duplicates = dupModule.duplicates = [dupModule, module]);
}
} else {
modulesByHash[hash] = module;
}
});
});
compilation.plugin("after-optimize-chunks", function(chunks) {
var entryChunks = chunks.filter(function(c) { return c.entry; });
entryChunks.forEach(function(chunk) {
(function x(dups, roots, visited, chunk) {
var currentDups = [];
var currentRoots = [];
chunk.modules.forEach(function(module) {
if(module.duplicates) {
var idx = currentDups.indexOf(module.duplicates);
if(idx >= 0) {
module.rootDuplicates = currentRoots[idx];
module.rootDuplicates.push(module);
module.rootDuplicates.commonModules = mergeCommonModules(module.rootDuplicates.commonModules, module.getAllModuleDependencies());
} else {
idx = dups.indexOf(module.duplicates);
if(idx < 0) {
module.rootDuplicates = [module];
module.rootDuplicates.commonModules = module.getAllModuleDependencies();
module.rootDuplicates.initialCcommonModulesLength = module.rootDuplicates.commonModules.length;
dups = dups.concat([module.duplicates]);
roots = roots.concat([module.rootDuplicates]);
currentDups = currentDups.concat([module.duplicates]);
currentRoots = currentRoots.concat([module.rootDuplicates]);
} else {
module.rootDuplicates = roots[idx];
module.rootDuplicates.commonModules = mergeCommonModules(module.rootDuplicates.commonModules, module.getAllModuleDependencies());
}
}
}
});
chunk.chunks.forEach(function(chunk) {
if(visited.indexOf(chunk) < 0)
x(dups, roots, visited.concat(chunk), chunk);
})
currentRoots.forEach(function(roots) {
var commonModules = roots.commonModules;
var initialLength = roots.initialCcommonModulesLength;
if(initialLength !== commonModules.length) {
var template = roots[0].createTemplate(commonModules);
roots.template = template;
chunk.addModule(template);
template.addChunk(chunk);
compilation.modules.push(template);
}
});
}([], [], [], chunk));
});
});
function mergeCommonModules(commonModules, newModules) {
return commonModules.filter(function(module) {
return newModules.indexOf(module) >= 0;
});
}
});
compiler.moduleTemplate = new DedupModuleTemplateDecorator(compiler.moduleTemplate);
compiler.mainTemplate.renderAddModule = function(hash, chunk, varModuleId, varModule) {
return [
"var _m = " + varModule + ";",
"switch(typeof _m) {",
"case \"number\":",
this.indent([
"modules[" + varModuleId + "] = modules[_m];",
"break;"
]),
"case \"object\":",
this.indent([
"modules[" + varModuleId + "] = (function(_m) {",
this.indent([
"var args = _m.slice(1), fn = modules[_m[0]];",
"return function (a,b,c) {",
this.indent([
"fn.apply(null, [a,b,c].concat(args));"
]),
"};"
]),
"}(_m));",
"break;"
]),
"default:",
this.indent("modules[" + varModuleId + "] = _m;"),
"}"
]
};
var oldRenderModules = compiler.mainTemplate.renderModules;
compiler.mainTemplate.renderModules = function renderModules(hash, chunk, moduleTemplate, dependencyTemplates) {
var source = new ConcatSource();
source.add("(function(modules) {\n");
source.add(this.indent([
"for(var i in modules) {",
this.indent([
"switch(typeof modules[i]) {",
"case \"number\":",
this.indent([
"modules[i] = modules[modules[i]];",
"break;"
]),
"case \"object\":",
this.indent([
"modules[i] = (function(_m) {",
this.indent([
"var args = _m.slice(1), fn = modules[_m[0]];",
"return function (a,b,c) {",
this.indent([
"fn.apply(null, [a,b,c].concat(args));"
]),
"};"
]),
"}(modules[i]));"
]),
"}"
]),
"}",
"return modules;"
]));
source.add("\n}(");
source.add(oldRenderModules.call(this, hash, chunk, moduleTemplate, dependencyTemplates));
source.add("))");
return source;
};
};
function DedupModuleTemplateDecorator(template) {
this.template = template;
}
DedupModuleTemplateDecorator.prototype.render = function(module, dependencyTemplates) {
if(!module.rootDuplicates) return this.template.render(module, dependencyTemplates);
if(module.rootDuplicates.template) {
module.rootDuplicates.template.addReason(module, {
type: "template",
request: module.request,
templateModules: module.rootDuplicates.template.templateModules
});
var array = [module.rootDuplicates.template.id].concat(module.getTemplateArguments(module.rootDuplicates.template.templateModules).map(function(module) {
if(typeof module.id !== "number")
return "(function webpackMissingModule() { throw new Error(" + JSON.stringify("Cannot find module") + "); }())"
return module.id;
}));
var source = new ConcatSource("[" + array.join(", ") + "]");
return source;
} else {
module.rootDuplicates.sort(function(a, b) {
return a.id - b.id;
});
if(module === module.rootDuplicates[0]) return this.template.render(module, dependencyTemplates);
var source = new ConcatSource("" + module.rootDuplicates[0].id);
return source;
}
};
DedupModuleTemplateDecorator.prototype.updateHash = function(hash) {
hash.update("DedupModuleTemplateDecorator");
this.template.updateHash(hash);
};

5
lib/webworker/WebWorkerMainTemplate.js

@ -50,8 +50,9 @@ WebWorkerMainTemplate.prototype.renderInit = function(hash, chunk) {
buf.push(
"this[" + JSON.stringify(chunkCallbackName) + "] = function webpackChunkCallback(chunkIds, moreModules) {",
this.indent([
"for(var moduleId in moreModules)",
this.indent("modules[moduleId] = moreModules[moduleId];"),
"for(var moduleId in moreModules) {",
this.indent(this.renderAddModule(hash, chunk, "moduleId", "moreModules[moduleId]")),
"}",
"while(chunkIds.length)",
this.indent("installedChunks[chunkIds.pop()] = 1;")
]),

2
package.json

@ -1,6 +1,6 @@
{
"name": "webpack",
"version": "0.10.0-beta19",
"version": "0.10.0-beta20",
"author": "Tobias Koppers @sokra",
"description": "Packs CommonJs/AMD/Labeled Modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loaders to preprocess files, i.e. json, jade, coffee, css, less, ... and your custom stuff.",
"dependencies": {

4
test/browsertest/build.js

@ -34,9 +34,9 @@ var library1 = cp.spawn("node", join(["../../bin/webpack.js", "--output-pathinfo
bindOutput(library1);
library1.on("exit", function(code) {
if(code === 0) {
// node ../../bin/webpack --output-pathinfo --colors --resolve-alias vm=vm-browserify --output-public-path js/ --module-bind json --module-bind css=style!css --module-bind less=style!css!less --module-bind coffee --module-bind jade --prefetch ./less/stylesheet.less ./lib/index js/web.js
// node ../../bin/webpack --output-pathinfo --colors --resolve-alias vm=vm-browserify --output-public-path js/ --module-bind json --module-bind css=style!css --module-bind less=style!css!less --module-bind coffee --module-bind jade --prefetch ./less/stylesheet.less --optimize-dedupe ./lib/index js/web.js
var main = cp.spawn("node", join(["../../bin/webpack.js", "--output-pathinfo", "--colors", "--resolve-alias", "vm=vm-browserify", "--workers",
"--output-public-path", "js/", "--module-bind", "json", "--module-bind", "css=style!css", "--module-bind", "less=style/url!file?postfix=.css&string!less", "--module-bind", "coffee", "--module-bind", "jade", "--prefetch", "./less/stylesheet.less", "./lib/index", "js/web.js", "--progress"], extraArgs));
"--output-public-path", "js/", "--module-bind", "json", "--module-bind", "css=style!css", "--module-bind", "less=style/url!file?postfix=.css&string!less", "--module-bind", "coffee", "--module-bind", "jade", "--prefetch", "./less/stylesheet.less", "--optimize-dedupe", "./lib/index", "js/web.js", "--progress"], extraArgs));
bindOutput(main);
}
});

1
test/browsertest/lib/dedupe1/dupdep.js

@ -0,0 +1 @@
module.exports = "edupe1";

1
test/browsertest/lib/dedupe1/index.js

@ -0,0 +1 @@
module.exports = require("../d") + require("./dupdep");

1
test/browsertest/lib/dedupe2/dupdep.js

@ -0,0 +1 @@
module.exports = "edupe2";

1
test/browsertest/lib/dedupe2/index.js

@ -0,0 +1 @@
module.exports = require("../d") + require("./dupdep");

9
test/browsertest/lib/index.web.js

@ -550,4 +550,13 @@ describe("main", function() {
});
});
describe("deduplication", function() {
it("should load a duplicate module with different dependencies correctly", function() {
var dedupe1 = require("./dedupe1");
var dedupe2 = require("./dedupe2");
dedupe1.should.be.eql("dedupe1");
dedupe2.should.be.eql("dedupe2");
});
});
});

Loading…
Cancel
Save