opensourcemirror
/
webpack
mirror of https://github.com/webpack/webpack.git
1
0
Fork 0
Code Issues Releases Wiki Activity

added experimental deduplication support #47

This commit is contained in:
Tobias Koppers 2013-06-12 16:16:06 +02:00
parent 7df05c7ddc
commit c30f16f439
22 changed files with 361 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/config-optimist.js
View File

@ -61,6 +61,8 @@ module.exports = function(optimist) {
.boolean("optimize-occurence-order").describe("optimize-occurence-order")
.boolean("optimize-dedupe").describe("optimize-dedupe")
.string("prefetch").describe("prefetch")
.string("provide").describe("provide")

5
bin/convert-argv.js
View File

@ -267,6 +267,11 @@ module.exports = function(optimist, argv, convertOptions) {
options.optimize.occurenceOrder = true;
});
ifBooleanArg("optimize-dedupe", function() {
ensureObject(options, "optimize");
options.optimize.dedupe = true;
});
ifArg("prefetch", function(request) {
ensureArray(options, "prefetch");
options.prefetch.push(request);

2
lib/FunctionModuleTemplate.js
View File

@ -19,7 +19,7 @@ FunctionModuleTemplate.prototype.render = function(module, dependencyTemplates)
source.add(" !*** " + req.replace(/\*\//g, "*_/") + " ***!\n");
source.add(" \\****" + req.replace(/./g, "*") + "****/\n");
}
source.add("/***/ function(module, exports, require) {\n\n");
source.add("/***/ function(" + ["module", "exports", "require"].concat(module.arguments || []).join(", ") + ") {\n\n");
source.add(new PrefixSource("\t", module.source(dependencyTemplates, this.outputOptions, this.requestShortener)));
source.add("\n\n/***/ }");
return source;

5
lib/JsonpMainTemplate.js
View File

@ -84,8 +84,9 @@ JsonpMainTemplate.prototype.renderInit = function(hash, chunk) {
"installedChunks[chunkId] = 0;"
]),
"}",
"for(moduleId in moreModules)",
this.indent("modules[moduleId] = moreModules[moduleId];"),
"for(moduleId in moreModules) {",
this.indent(this.renderAddModule(hash, chunk, "moduleId", "moreModules[moduleId]")),
"}",
"while(callbacks.length)",
this.indent("callbacks.shift().call(null, require);"),
]),

18
lib/MainTemplate.js
View File

@ -33,7 +33,16 @@ MainTemplate.prototype.render = function(hash, chunk, moduleTemplate, dependency
source.add(new PrefixSource("/******/ \t", new OriginalSource(this.asString(buf), "webpackBootstrap " + hash)));
source.add("\n/******/ })\n");
source.add("/************************************************************************/\n");
source.add("/******/ ({\n");
source.add("/******/ (");
source.add(this.renderModules(hash, chunk, moduleTemplate, dependencyTemplates));
source.add(")");
chunk.rendered = true;
return source;
};
MainTemplate.prototype.renderModules = function renderModules(hash, chunk, moduleTemplate, dependencyTemplates) {
var source = new ConcatSource();
source.add("{\n");
source.add(this.asString(this.renderInitModules(hash, chunk, moduleTemplate, dependencyTemplates)));
source.add("\n");
chunk.modules.forEach(function(module, idx) {
@ -41,8 +50,7 @@ MainTemplate.prototype.render = function(hash, chunk, moduleTemplate, dependency
source.add("\n/***/ " + module.id + ":\n");
source.add(moduleTemplate.render(module, dependencyTemplates));
});
source.add("\n/******/ })");
chunk.rendered = true;
source.add("\n/******/ }");
return source;
};
@ -138,6 +146,10 @@ MainTemplate.prototype.renderInitModules = function(hash, chunk, moduleTemplate,
];
};
MainTemplate.prototype.renderAddModule = function(hash, chunk, varModuleId, varModule) {
return ["modules[" + varModuleId + "] = " + varModule + ";"]
}
MainTemplate.prototype.updateHash = function(hash) {
hash.update("maintemplate");
hash.update("1");

4
lib/Module.js
View File

@ -23,10 +23,6 @@ module.exports = Module;
Module.prototype = Object.create(DependenciesBlock.prototype);
Module.prototype.separable = function(callback) {
callback(false);
};
Module.prototype.disconnect = function() {
this.reasons.length = 0;
this.lastId = this.id;

82
lib/NormalModule.js
View File

@ -9,6 +9,7 @@ var OriginalSource = require("webpack-core/lib/OriginalSource");
var RawSource = require("webpack-core/lib/RawSource");
var ReplaceSource = require("webpack-core/lib/ReplaceSource");
var ModuleParseError = require("./ModuleParseError");
var TemplateArgumentDependency = require("./dependencies/TemplateArgumentDependency");
var path = require("path");
function NormalModule(request, userRequest, rawRequest, loaders, resource, parser) {
@ -82,7 +83,7 @@ NormalModule.prototype.source = function(dependencyTemplates, outputOptions, req
function doDep(dep) {
var template = dependencyTemplates.get(dep.Class);
if(!template) throw new Error("No template for dependency: " + dep.Class.name);
template.apply(dep, source, outputOptions, requestShortener);
template.apply(dep, source, outputOptions, requestShortener, dependencyTemplates);
}
function doVariable(vars, variable) {
var name = variable.name;
@ -151,3 +152,82 @@ NormalModule.prototype.updateHash = function(hash) {
hash.update("null");
Module.prototype.updateHash.call(this, hash);
};
NormalModule.prototype.getSourceHash = function() {
if(!this._source) return "";
var hash = new (require("crypto").Hash)("md5");
hash.update(this._source.source());
return hash.digest("hex");
};
NormalModule.prototype.getAllModuleDependencies = function() {
var list = [];
function doDep(dep) {
if(dep.module && list.indexOf(dep.module) < 0)
list.push(dep.module);
}
function doVariable(variable) {
variable.dependencies.forEach(doDep);
}
function doBlock(block) {
block.variables.forEach(doVariable);
block.dependencies.forEach(doDep);
block.blocks.forEach(doBlock);
}
doBlock(this);
return list;
};
NormalModule.prototype.createTemplate = function(keepModules) {
var template = new NormalModule("", "", "", [], "", null);
template._source = this._source;
template.built = this.built;
template.templateModules = keepModules;
var args = template.arguments = [];
function doDeps(deps) {
return deps.map(function(dep) {
if(keepModules.indexOf(dep.module) < 0) {
var argName = "__webpack_module_template_argument_" + args.length + "__";
args.push(argName);
return new TemplateArgumentDependency(argName, dep);
} else {
return dep;
}
});
}
function doVariable(variable, newVariable) {
variable.dependencies.forEach(doDep);
}
function doBlock(block, newBlock) {
block.variables.forEach(function(variable) {
var newDependencies = doDeps(variable.dependencies);
newBlock.addVariable(variable.name, variable.expression, newDependencies);
});
newBlock.dependencies = doDeps(block.dependencies);
block.blocks.forEach(function(childBlock) {
var newChildBlock = new AsyncDependenciesBlock(childBlock.name);
newBlock.addBlock(newChildBlock);
doBlock(childBlock, newChildBlock);
});
}
doBlock(this, template);
return template;
};
NormalModule.prototype.getTemplateArguments = function(keepModules) {
var list = [];
function doDep(dep) {
if(dep.module && keepModules.indexOf(dep.module) < 0)
list.push(dep.module);
}
function doVariable(variable) {
variable.dependencies.forEach(doDep);
}
function doBlock(block) {
block.variables.forEach(doVariable);
block.dependencies.forEach(doDep);
block.blocks.forEach(doBlock);
}
doBlock(this);
return list;
};

3
lib/Stats.js
View File

@ -135,6 +135,7 @@ Stats.prototype.toJson = function toJson(options, forToString) {
userRequest: reason.dependency.userRequest
}
var dep = reason.dependency;
if(dep.templateModules) obj.templateModules = dep.templateModules.map(function(module) { return module.id; });
if(dep.loc) obj.loc = dep.loc.start.line + ":" + dep.loc.start.column + "-" +
(dep.loc.start.line != dep.loc.end.line ? dep.loc.end.line + ":" : "") + dep.loc.end.column
return obj;
@ -436,6 +437,7 @@ Stats.jsonToString = function jsonToString(obj, useColors) {
normal(reason.type);
normal(" ");
cyan(reason.userRequest);
if(reason.templateModules) cyan(reason.templateModules.join(" "));
normal(" [");
normal(reason.moduleId);
normal("] ");
@ -469,6 +471,7 @@ Stats.jsonToString = function jsonToString(obj, useColors) {
normal(reason.type);
normal(" ");
cyan(reason.userRequest);
if(reason.templateModules) cyan(reason.templateModules.join(" "));
normal(" [");
normal(reason.moduleId);
normal("] ");

4
lib/WebpackOptionsApply.js
View File

@ -42,6 +42,7 @@ var RemoveParentModulesPlugin = require("./optimize/RemoveParentModulesPlugin");
var RemoveEmptyChunksPlugin = require("./optimize/RemoveEmptyChunksPlugin");
var MergeDuplicateChunksPlugin = require("./optimize/MergeDuplicateChunksPlugin");
var FlagIncludedChunksPlugin = require("./optimize/FlagIncludedChunksPlugin");
var DedupePlugin = require("./optimize/DedupePlugin");
var ModulesInDirectoriesPlugin = require("enhanced-resolve/lib/ModulesInDirectoriesPlugin");
var ModulesInRootPlugin = require("enhanced-resolve/lib/ModulesInRootPlugin");
@ -156,6 +157,9 @@ WebpackOptionsApply.prototype.process = function(options, compiler) {
else if(options.optimize.minimize)
compiler.apply(new UglifyJsPlugin(options.optimize.minimize));
if(options.optimize.dedupe === true)
compiler.apply(new DedupePlugin());
if(options.cache === undefined ? options.watch : options.cache)
compiler.apply(new CachePlugin(typeof options.cache == "object" ? options.cache : null));

4
lib/dependencies/ModuleDependencyTemplateAsId.js
View File

@ -14,3 +14,7 @@ ModuleDependencyTemplateAsId.prototype.apply = function(dep, source, outputOptio
var content = "(function webpackMissingModule() { throw new Error(" + JSON.stringify("Cannot find module \"" + dep.request + "\"") + "); }())";
source.replace(dep.range[0], dep.range[1]-1, content);
};
ModuleDependencyTemplateAsId.prototype.applyAsTemplateArgument = function(name, dep, source, outputOptions, requestShortener) {
source.replace(dep.range[0], dep.range[1]-1, name);
};

4
lib/dependencies/ModuleDependencyTemplateAsRequireId.js
View File

@ -14,3 +14,7 @@ ModuleDependencyTemplateAsRequireId.prototype.apply = function(dep, source, outp
var content = "(function webpackMissingModule() { throw new Error(" + JSON.stringify("Cannot find module \"" + dep.request + "\"") + "); }())";
source.replace(dep.range[0], dep.range[1]-1, content);
};
ModuleDependencyTemplateAsRequireId.prototype.applyAsTemplateArgument = function(name, dep, source, outputOptions, requestShortener) {
source.replace(dep.range[0], dep.range[1]-1, "(require(" + name + "))");
};

27
lib/dependencies/TemplateArgumentDependency.js Normal file
View File

@ -0,0 +1,27 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
function TemplateArgumentDependency(name, dep) {
this.name = name;
this.Class = TemplateArgumentDependency;
this.dep = dep;
}
module.exports = TemplateArgumentDependency;
TemplateArgumentDependency.prototype.type = "template argument";
TemplateArgumentDependency.prototype.updateHash = function(hash) {
hash.update(this.name);
};
TemplateArgumentDependency.Template = function TemplateArgumentDependencyTemplate() {};
TemplateArgumentDependency.Template.prototype.apply = function(dep, source, outputOptions, requestShortener, dependencyTemplates) {
var d = dep.dep;
var template = dependencyTemplates.get(d.Class);
if(!template) throw new Error("No template for dependency: " + d.Class.name);
if(!template.applyAsTemplateArgument) throw new Error("Template cannot be applied as TemplateArgument: " + d.Class.name);
return template.applyAsTemplateArgument(dep.name, d, source, outputOptions, requestShortener, dependencyTemplates);
};

5
lib/node/NodeMainTemplate.js
View File

@ -39,8 +39,9 @@ NodeMainTemplate.prototype.renderRequireEnsure = function(hash, chunk) {
.replace(Template.REGEXP_NAME, ""))
.replace(Template.REGEXP_ID, "\" + chunkId + \"") + ");",
"var moreModules = chunk.modules, chunkIds = chunk.ids;",
"for(var moduleId in moreModules)",
this.indent("modules[moduleId] = moreModules[moduleId];"),
"for(var moduleId in moreModules) {",
this.indent(this.renderAddModule(hash, chunk, "moduleId", "moreModules[moduleId]")),
"}",
"for(var i = 0; i < chunkIds.length; i++)",
this.indent("installedChunks[chunkIds[i]] = 1;"),
]),

190
lib/optimize/DedupePlugin.js Normal file
View File

@ -0,0 +1,190 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
var ConcatSource = require("webpack-core/lib/ConcatSource");
var TemplateArgumentDependency = require("../dependencies/TemplateArgumentDependency");
function DedupePlugin() {
}
module.exports = DedupePlugin;
DedupePlugin.prototype.apply = function(compiler) {
compiler.plugin("compilation", function(compilation) {
compilation.dependencyTemplates.set(TemplateArgumentDependency, new TemplateArgumentDependency.Template());
compilation.plugin("after-optimize-modules", function(modules) {
var modulesByHash = {};
var allDups = [];
modules.forEach(function(module, idx) {
if(!module.getSourceHash || !module.getAllModuleDependencies || !module.createTemplate || !module.getTemplateArguments) return;
var hash = module.getSourceHash();
var dupModule = modulesByHash[hash];
if(dupModule) {
if(dupModule.duplicates) {
dupModule.duplicates.push(module);
module.duplicates = dupModule.duplicates;
} else {
allDups.push(module.duplicates = dupModule.duplicates = [dupModule, module]);
}
} else {
modulesByHash[hash] = module;
}
});
});
compilation.plugin("after-optimize-chunks", function(chunks) {
var entryChunks = chunks.filter(function(c) { return c.entry; });
entryChunks.forEach(function(chunk) {
(function x(dups, roots, visited, chunk) {
var currentDups = [];
var currentRoots = [];
chunk.modules.forEach(function(module) {
if(module.duplicates) {
var idx = currentDups.indexOf(module.duplicates);
if(idx >= 0) {
module.rootDuplicates = currentRoots[idx];
module.rootDuplicates.push(module);
module.rootDuplicates.commonModules = mergeCommonModules(module.rootDuplicates.commonModules, module.getAllModuleDependencies());
} else {
idx = dups.indexOf(module.duplicates);
if(idx < 0) {
module.rootDuplicates = [module];
module.rootDuplicates.commonModules = module.getAllModuleDependencies();
module.rootDuplicates.initialCcommonModulesLength = module.rootDuplicates.commonModules.length;
dups = dups.concat([module.duplicates]);
roots = roots.concat([module.rootDuplicates]);
currentDups = currentDups.concat([module.duplicates]);
currentRoots = currentRoots.concat([module.rootDuplicates]);
} else {
module.rootDuplicates = roots[idx];
module.rootDuplicates.commonModules = mergeCommonModules(module.rootDuplicates.commonModules, module.getAllModuleDependencies());
}
}
}
});
chunk.chunks.forEach(function(chunk) {
if(visited.indexOf(chunk) < 0)
x(dups, roots, visited.concat(chunk), chunk);
})
currentRoots.forEach(function(roots) {
var commonModules = roots.commonModules;
var initialLength = roots.initialCcommonModulesLength;
if(initialLength !== commonModules.length) {
var template = roots[0].createTemplate(commonModules);
roots.template = template;
chunk.addModule(template);
template.addChunk(chunk);
compilation.modules.push(template);
}
});
}([], [], [], chunk));
});
});
function mergeCommonModules(commonModules, newModules) {
return commonModules.filter(function(module) {
return newModules.indexOf(module) >= 0;
});
}
});
compiler.moduleTemplate = new DedupModuleTemplateDecorator(compiler.moduleTemplate);
compiler.mainTemplate.renderAddModule = function(hash, chunk, varModuleId, varModule) {
return [
"var _m = " + varModule + ";",
"switch(typeof _m) {",
"case \"number\":",
this.indent([
"modules[" + varModuleId + "] = modules[_m];",
"break;"
]),
"case \"object\":",
this.indent([
"modules[" + varModuleId + "] = (function(_m) {",
this.indent([
"var args = _m.slice(1), fn = modules[_m[0]];",
"return function (a,b,c) {",
this.indent([
"fn.apply(null, [a,b,c].concat(args));"
]),
"};"
]),
"}(_m));",
"break;"
]),
"default:",
this.indent("modules[" + varModuleId + "] = _m;"),
"}"
]
};
var oldRenderModules = compiler.mainTemplate.renderModules;
compiler.mainTemplate.renderModules = function renderModules(hash, chunk, moduleTemplate, dependencyTemplates) {
var source = new ConcatSource();
source.add("(function(modules) {\n");
source.add(this.indent([
"for(var i in modules) {",
this.indent([
"switch(typeof modules[i]) {",
"case \"number\":",
this.indent([
"modules[i] = modules[modules[i]];",
"break;"
]),
"case \"object\":",
this.indent([
"modules[i] = (function(_m) {",
this.indent([
"var args = _m.slice(1), fn = modules[_m[0]];",
"return function (a,b,c) {",
this.indent([
"fn.apply(null, [a,b,c].concat(args));"
]),
"};"
]),
"}(modules[i]));"
]),
"}"
]),
"}",
"return modules;"
]));
source.add("\n}(");
source.add(oldRenderModules.call(this, hash, chunk, moduleTemplate, dependencyTemplates));
source.add("))");
return source;
};
};
function DedupModuleTemplateDecorator(template) {
this.template = template;
}
DedupModuleTemplateDecorator.prototype.render = function(module, dependencyTemplates) {
if(!module.rootDuplicates) return this.template.render(module, dependencyTemplates);
if(module.rootDuplicates.template) {
module.rootDuplicates.template.addReason(module, {
type: "template",
request: module.request,
templateModules: module.rootDuplicates.template.templateModules
});
var array = [module.rootDuplicates.template.id].concat(module.getTemplateArguments(module.rootDuplicates.template.templateModules).map(function(module) {
if(typeof module.id !== "number")
return "(function webpackMissingModule() { throw new Error(" + JSON.stringify("Cannot find module") + "); }())"
return module.id;
}));
var source = new ConcatSource("[" + array.join(", ") + "]");
return source;
} else {
module.rootDuplicates.sort(function(a, b) {
return a.id - b.id;
});
if(module === module.rootDuplicates[0]) return this.template.render(module, dependencyTemplates);
var source = new ConcatSource("" + module.rootDuplicates[0].id);
return source;
}
};
DedupModuleTemplateDecorator.prototype.updateHash = function(hash) {
hash.update("DedupModuleTemplateDecorator");
this.template.updateHash(hash);
};

5
lib/webworker/WebWorkerMainTemplate.js
View File

@ -50,8 +50,9 @@ WebWorkerMainTemplate.prototype.renderInit = function(hash, chunk) {
buf.push(
"this[" + JSON.stringify(chunkCallbackName) + "] = function webpackChunkCallback(chunkIds, moreModules) {",
this.indent([
"for(var moduleId in moreModules)",
this.indent("modules[moduleId] = moreModules[moduleId];"),
"for(var moduleId in moreModules) {",
this.indent(this.renderAddModule(hash, chunk, "moduleId", "moreModules[moduleId]")),
"}",
"while(chunkIds.length)",
this.indent("installedChunks[chunkIds.pop()] = 1;")
]),

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "webpack",
"version": "0.10.0-beta19",
"version": "0.10.0-beta20",
"author": "Tobias Koppers @sokra",
"description": "Packs CommonJs/AMD/Labeled Modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loaders to preprocess files, i.e. json, jade, coffee, css, less, ... and your custom stuff.",
"dependencies": {

4
test/browsertest/build.js
View File

@ -34,9 +34,9 @@ var library1 = cp.spawn("node", join(["../../bin/webpack.js", "--output-pathinfo
bindOutput(library1);
library1.on("exit", function(code) {
if(code === 0) {
// node ../../bin/webpack --output-pathinfo --colors --resolve-alias vm=vm-browserify --output-public-path js/ --module-bind json --module-bind css=style!css --module-bind less=style!css!less --module-bind coffee --module-bind jade --prefetch ./less/stylesheet.less ./lib/index js/web.js
// node ../../bin/webpack --output-pathinfo --colors --resolve-alias vm=vm-browserify --output-public-path js/ --module-bind json --module-bind css=style!css --module-bind less=style!css!less --module-bind coffee --module-bind jade --prefetch ./less/stylesheet.less --optimize-dedupe ./lib/index js/web.js
var main = cp.spawn("node", join(["../../bin/webpack.js", "--output-pathinfo", "--colors", "--resolve-alias", "vm=vm-browserify", "--workers",
"--output-public-path", "js/", "--module-bind", "json", "--module-bind", "css=style!css", "--module-bind", "less=style/url!file?postfix=.css&string!less", "--module-bind", "coffee", "--module-bind", "jade", "--prefetch", "./less/stylesheet.less", "./lib/index", "js/web.js", "--progress"], extraArgs));
"--output-public-path", "js/", "--module-bind", "json", "--module-bind", "css=style!css", "--module-bind", "less=style/url!file?postfix=.css&string!less", "--module-bind", "coffee", "--module-bind", "jade", "--prefetch", "./less/stylesheet.less", "--optimize-dedupe", "./lib/index", "js/web.js", "--progress"], extraArgs));
bindOutput(main);
}
});

1
test/browsertest/lib/dedupe1/dupdep.js Normal file
View File

@ -0,0 +1 @@
module.exports = "edupe1";

1
test/browsertest/lib/dedupe1/index.js Normal file
View File

@ -0,0 +1 @@
module.exports = require("../d") + require("./dupdep");

1
test/browsertest/lib/dedupe2/dupdep.js Normal file
View File

@ -0,0 +1 @@
module.exports = "edupe2";

1
test/browsertest/lib/dedupe2/index.js Normal file
View File

@ -0,0 +1 @@
module.exports = require("../d") + require("./dupdep");

9
test/browsertest/lib/index.web.js
View File

@ -550,4 +550,13 @@ describe("main", function() {
});
});
describe("deduplication", function() {
it("should load a duplicate module with different dependencies correctly", function() {
var dedupe1 = require("./dedupe1");
var dedupe2 = require("./dedupe2");
dedupe1.should.be.eql("dedupe1");
dedupe2.should.be.eql("dedupe2");
});
});
});