added experimental deduplication support #47

bin/config-optimist.js

@@ -61,6 +61,8 @@ module.exports = function(optimist) {
 
 	.boolean("optimize-occurence-order").describe("optimize-occurence-order")
 
+	.boolean("optimize-dedupe").describe("optimize-dedupe")
+
 	.string("prefetch").describe("prefetch")
 
 	.string("provide").describe("provide")

bin/convert-argv.js

@@ -267,6 +267,11 @@ module.exports = function(optimist, argv, convertOptions) {
 		options.optimize.occurenceOrder = true;
 	});
 
+	ifBooleanArg("optimize-dedupe", function() {
+		ensureObject(options, "optimize");
+		options.optimize.dedupe = true;
+	});
+
 	ifArg("prefetch", function(request) {
 		ensureArray(options, "prefetch");
 		options.prefetch.push(request);

lib/FunctionModuleTemplate.js

@@ -19,7 +19,7 @@ FunctionModuleTemplate.prototype.render = function(module, dependencyTemplates)
 		source.add("  !*** " + req.replace(/\*\//g, "*_/") + " ***!\n");
 		source.add("  \\****" + req.replace(/./g, "*") + "****/\n");
 	}
-	source.add("/***/ function(module, exports, require) {\n\n");
+	source.add("/***/ function(" + ["module", "exports", "require"].concat(module.arguments || []).join(", ") + ") {\n\n");
 	source.add(new PrefixSource("\t", module.source(dependencyTemplates, this.outputOptions, this.requestShortener)));
 	source.add("\n\n/***/ }");
 	return source;

lib/JsonpMainTemplate.js

@@ -84,8 +84,9 @@ JsonpMainTemplate.prototype.renderInit = function(hash, chunk) {
 					"installedChunks[chunkId] = 0;"
 				]),
 				"}",
-				"for(moduleId in moreModules)",
+				"for(moduleId in moreModules) {",
-				this.indent("modules[moduleId] = moreModules[moduleId];"),
+				this.indent(this.renderAddModule(hash, chunk, "moduleId", "moreModules[moduleId]")),
+				"}",
 				"while(callbacks.length)",
 				this.indent("callbacks.shift().call(null, require);"),
 			]),

lib/MainTemplate.js

@@ -33,7 +33,16 @@ MainTemplate.prototype.render = function(hash, chunk, moduleTemplate, dependency
 	source.add(new PrefixSource("/******/ \t", new OriginalSource(this.asString(buf), "webpackBootstrap " + hash)));
 	source.add("\n/******/ })\n");
 	source.add("/************************************************************************/\n");
-	source.add("/******/ ({\n");
+	source.add("/******/ (");
+	source.add(this.renderModules(hash, chunk, moduleTemplate, dependencyTemplates));
+	source.add(")");
+	chunk.rendered = true;
+	return source;
+};
+
+MainTemplate.prototype.renderModules = function renderModules(hash, chunk, moduleTemplate, dependencyTemplates) {
+	var source = new ConcatSource();
+	source.add("{\n");
 	source.add(this.asString(this.renderInitModules(hash, chunk, moduleTemplate, dependencyTemplates)));
 	source.add("\n");
 	chunk.modules.forEach(function(module, idx) {
@@ -41,8 +50,7 @@ MainTemplate.prototype.render = function(hash, chunk, moduleTemplate, dependency
 		source.add("\n/***/ " + module.id + ":\n");
 		source.add(moduleTemplate.render(module, dependencyTemplates));
 	});
-	source.add("\n/******/ })");
+	source.add("\n/******/ }");
-	chunk.rendered = true;
 	return source;
 };
 
@@ -138,6 +146,10 @@ MainTemplate.prototype.renderInitModules = function(hash, chunk, moduleTemplate,
 	];
 };
 
+MainTemplate.prototype.renderAddModule = function(hash, chunk, varModuleId, varModule) {
+	return ["modules[" + varModuleId + "] = " + varModule + ";"]
+}
+
 MainTemplate.prototype.updateHash = function(hash) {
 	hash.update("maintemplate");
 	hash.update("1");

lib/Module.js

@@ -23,10 +23,6 @@ module.exports = Module;
 
 Module.prototype = Object.create(DependenciesBlock.prototype);
 
-Module.prototype.separable = function(callback) {
-	callback(false);
-};
-
 Module.prototype.disconnect = function() {
 	this.reasons.length = 0;
 	this.lastId = this.id;

lib/NormalModule.js

@@ -9,6 +9,7 @@ var OriginalSource = require("webpack-core/lib/OriginalSource");
 var RawSource = require("webpack-core/lib/RawSource");
 var ReplaceSource = require("webpack-core/lib/ReplaceSource");
 var ModuleParseError = require("./ModuleParseError");
+var TemplateArgumentDependency = require("./dependencies/TemplateArgumentDependency");
 var path = require("path");
 
 function NormalModule(request, userRequest, rawRequest, loaders, resource, parser) {
@@ -82,7 +83,7 @@ NormalModule.prototype.source = function(dependencyTemplates, outputOptions, req
 	function doDep(dep) {
 		var template = dependencyTemplates.get(dep.Class);
 		if(!template) throw new Error("No template for dependency: " + dep.Class.name);
-		template.apply(dep, source, outputOptions, requestShortener);
+		template.apply(dep, source, outputOptions, requestShortener, dependencyTemplates);
 	}
 	function doVariable(vars, variable) {
 		var name = variable.name;
@@ -151,3 +152,82 @@ NormalModule.prototype.updateHash = function(hash) {
 		hash.update("null");
 	Module.prototype.updateHash.call(this, hash);
 };
+
+NormalModule.prototype.getSourceHash = function() {
+	if(!this._source) return "";
+	var hash = new (require("crypto").Hash)("md5");
+	hash.update(this._source.source());
+	return hash.digest("hex");
+};
+
+NormalModule.prototype.getAllModuleDependencies = function() {
+	var list = [];
+	function doDep(dep) {
+		if(dep.module && list.indexOf(dep.module) < 0)
+			list.push(dep.module);
+	}
+	function doVariable(variable) {
+		variable.dependencies.forEach(doDep);
+	}
+	function doBlock(block) {
+		block.variables.forEach(doVariable);
+		block.dependencies.forEach(doDep);
+		block.blocks.forEach(doBlock);
+	}
+	doBlock(this);
+	return list;
+};
+
+NormalModule.prototype.createTemplate = function(keepModules) {
+	var template = new NormalModule("", "", "", [], "", null);
+	template._source = this._source;
+	template.built = this.built;
+	template.templateModules = keepModules;
+	var args = template.arguments = [];
+	function doDeps(deps) {
+		return deps.map(function(dep) {
+			if(keepModules.indexOf(dep.module) < 0) {
+				var argName = "__webpack_module_template_argument_" + args.length + "__";
+				args.push(argName);
+				return new TemplateArgumentDependency(argName, dep);
+			} else {
+				return dep;
+			}
+		});
+	}
+	function doVariable(variable, newVariable) {
+		variable.dependencies.forEach(doDep);
+	}
+	function doBlock(block, newBlock) {
+		block.variables.forEach(function(variable) {
+			var newDependencies = doDeps(variable.dependencies);
+			newBlock.addVariable(variable.name, variable.expression, newDependencies);
+		});
+		newBlock.dependencies = doDeps(block.dependencies);
+		block.blocks.forEach(function(childBlock) {
+			var newChildBlock = new AsyncDependenciesBlock(childBlock.name);
+			newBlock.addBlock(newChildBlock);
+			doBlock(childBlock, newChildBlock);
+		});
+	}
+	doBlock(this, template);
+	return template;
+};
+
+NormalModule.prototype.getTemplateArguments = function(keepModules) {
+	var list = [];
+	function doDep(dep) {
+		if(dep.module && keepModules.indexOf(dep.module) < 0)
+			list.push(dep.module);
+	}
+	function doVariable(variable) {
+		variable.dependencies.forEach(doDep);
+	}
+	function doBlock(block) {
+		block.variables.forEach(doVariable);
+		block.dependencies.forEach(doDep);
+		block.blocks.forEach(doBlock);
+	}
+	doBlock(this);
+	return list;
+};

lib/Stats.js

@@ -135,6 +135,7 @@ Stats.prototype.toJson = function toJson(options, forToString) {
 					userRequest: reason.dependency.userRequest
 				}
 				var dep = reason.dependency;
+				if(dep.templateModules) obj.templateModules = dep.templateModules.map(function(module) { return module.id; });
 				if(dep.loc) obj.loc = dep.loc.start.line + ":" + dep.loc.start.column + "-" +
 					(dep.loc.start.line != dep.loc.end.line ? dep.loc.end.line + ":" : "") + dep.loc.end.column
 				return obj;
@@ -436,6 +437,7 @@ Stats.jsonToString = function jsonToString(obj, useColors) {
 							normal(reason.type);
 							normal(" ");
 							cyan(reason.userRequest);
+							if(reason.templateModules) cyan(reason.templateModules.join(" "));
 							normal(" [");
 							normal(reason.moduleId);
 							normal("] ");
@@ -469,6 +471,7 @@ Stats.jsonToString = function jsonToString(obj, useColors) {
 					normal(reason.type);
 					normal(" ");
 					cyan(reason.userRequest);
+					if(reason.templateModules) cyan(reason.templateModules.join(" "));
 					normal(" [");
 					normal(reason.moduleId);
 					normal("] ");

lib/WebpackOptionsApply.js

@@ -42,6 +42,7 @@ var RemoveParentModulesPlugin = require("./optimize/RemoveParentModulesPlugin");
 var RemoveEmptyChunksPlugin = require("./optimize/RemoveEmptyChunksPlugin");
 var MergeDuplicateChunksPlugin = require("./optimize/MergeDuplicateChunksPlugin");
 var FlagIncludedChunksPlugin = require("./optimize/FlagIncludedChunksPlugin");
+var DedupePlugin = require("./optimize/DedupePlugin");
 
 var ModulesInDirectoriesPlugin = require("enhanced-resolve/lib/ModulesInDirectoriesPlugin");
 var ModulesInRootPlugin = require("enhanced-resolve/lib/ModulesInRootPlugin");
@@ -156,6 +157,9 @@ WebpackOptionsApply.prototype.process = function(options, compiler) {
 	else if(options.optimize.minimize)
 		compiler.apply(new UglifyJsPlugin(options.optimize.minimize));
 
+	if(options.optimize.dedupe === true)
+		compiler.apply(new DedupePlugin());
+
 	if(options.cache === undefined ? options.watch : options.cache)
 		compiler.apply(new CachePlugin(typeof options.cache == "object" ? options.cache : null));
 

lib/dependencies/ModuleDependencyTemplateAsId.js

@@ -14,3 +14,7 @@ ModuleDependencyTemplateAsId.prototype.apply = function(dep, source, outputOptio
 		var content = "(function webpackMissingModule() { throw new Error(" + JSON.stringify("Cannot find module \"" + dep.request + "\"") + "); }())";
 	source.replace(dep.range[0], dep.range[1]-1, content);
 };
+
+ModuleDependencyTemplateAsId.prototype.applyAsTemplateArgument = function(name, dep, source, outputOptions, requestShortener) {
+	source.replace(dep.range[0], dep.range[1]-1, name);
+};

lib/dependencies/ModuleDependencyTemplateAsRequireId.js

@@ -14,3 +14,7 @@ ModuleDependencyTemplateAsRequireId.prototype.apply = function(dep, source, outp
 		var content = "(function webpackMissingModule() { throw new Error(" + JSON.stringify("Cannot find module \"" + dep.request + "\"") + "); }())";
 	source.replace(dep.range[0], dep.range[1]-1, content);
 };
+
+ModuleDependencyTemplateAsRequireId.prototype.applyAsTemplateArgument = function(name, dep, source, outputOptions, requestShortener) {
+	source.replace(dep.range[0], dep.range[1]-1, "(require(" + name + "))");
+};

lib/dependencies/TemplateArgumentDependency.js

@@ -0,0 +1,27 @@
+/*
+	MIT License http://www.opensource.org/licenses/mit-license.php
+	Author Tobias Koppers @sokra
+*/
+function TemplateArgumentDependency(name, dep) {
+	this.name = name;
+	this.Class = TemplateArgumentDependency;
+	this.dep = dep;
+}
+module.exports = TemplateArgumentDependency;
+
+TemplateArgumentDependency.prototype.type = "template argument";
+
+TemplateArgumentDependency.prototype.updateHash = function(hash) {
+	hash.update(this.name);
+};
+
+
+TemplateArgumentDependency.Template = function TemplateArgumentDependencyTemplate() {};
+
+TemplateArgumentDependency.Template.prototype.apply = function(dep, source, outputOptions, requestShortener, dependencyTemplates) {
+	var d = dep.dep;
+	var template = dependencyTemplates.get(d.Class);
+	if(!template) throw new Error("No template for dependency: " + d.Class.name);
+	if(!template.applyAsTemplateArgument) throw new Error("Template cannot be applied as TemplateArgument: " + d.Class.name);
+	return template.applyAsTemplateArgument(dep.name, d, source, outputOptions, requestShortener, dependencyTemplates);
+};

lib/node/NodeMainTemplate.js

@@ -39,8 +39,9 @@ NodeMainTemplate.prototype.renderRequireEnsure = function(hash, chunk) {
 					.replace(Template.REGEXP_NAME, ""))
 				.replace(Template.REGEXP_ID, "\" + chunkId + \"") + ");",
 			"var moreModules = chunk.modules, chunkIds = chunk.ids;",
-			"for(var moduleId in moreModules)",
+			"for(var moduleId in moreModules) {",
-			this.indent("modules[moduleId] = moreModules[moduleId];"),
+			this.indent(this.renderAddModule(hash, chunk, "moduleId", "moreModules[moduleId]")),
+			"}",
 			"for(var i = 0; i < chunkIds.length; i++)",
 			this.indent("installedChunks[chunkIds[i]] = 1;"),
 		]),

lib/optimize/DedupePlugin.js

@@ -0,0 +1,190 @@
+/*
+	MIT License http://www.opensource.org/licenses/mit-license.php
+	Author Tobias Koppers @sokra
+*/
+var ConcatSource = require("webpack-core/lib/ConcatSource");
+var TemplateArgumentDependency = require("../dependencies/TemplateArgumentDependency");
+
+function DedupePlugin() {
+}
+module.exports = DedupePlugin;
+
+DedupePlugin.prototype.apply = function(compiler) {
+	compiler.plugin("compilation", function(compilation) {
+
+		compilation.dependencyTemplates.set(TemplateArgumentDependency, new TemplateArgumentDependency.Template());
+
+		compilation.plugin("after-optimize-modules", function(modules) {
+			var modulesByHash = {};
+			var allDups = [];
+			modules.forEach(function(module, idx) {
+				if(!module.getSourceHash || !module.getAllModuleDependencies || !module.createTemplate || !module.getTemplateArguments) return;
+				var hash = module.getSourceHash();
+				var dupModule = modulesByHash[hash];
+				if(dupModule) {
+					if(dupModule.duplicates) {
+						dupModule.duplicates.push(module);
+						module.duplicates = dupModule.duplicates;
+					} else {
+						allDups.push(module.duplicates = dupModule.duplicates = [dupModule, module]);
+					}
+				} else {
+					modulesByHash[hash] = module;
+				}
+			});
+		});
+		compilation.plugin("after-optimize-chunks", function(chunks) {
+			var entryChunks = chunks.filter(function(c) { return c.entry; });
+			entryChunks.forEach(function(chunk) {
+				(function x(dups, roots, visited, chunk) {
+					var currentDups = [];
+					var currentRoots = [];
+					chunk.modules.forEach(function(module) {
+						if(module.duplicates) {
+							var idx = currentDups.indexOf(module.duplicates);
+							if(idx >= 0) {
+								module.rootDuplicates = currentRoots[idx];
+								module.rootDuplicates.push(module);
+								module.rootDuplicates.commonModules = mergeCommonModules(module.rootDuplicates.commonModules, module.getAllModuleDependencies());
+							} else {
+								idx = dups.indexOf(module.duplicates);
+								if(idx < 0) {
+									module.rootDuplicates = [module];
+									module.rootDuplicates.commonModules = module.getAllModuleDependencies();
+									module.rootDuplicates.initialCcommonModulesLength = module.rootDuplicates.commonModules.length;
+									dups = dups.concat([module.duplicates]);
+									roots = roots.concat([module.rootDuplicates]);
+									currentDups = currentDups.concat([module.duplicates]);
+									currentRoots = currentRoots.concat([module.rootDuplicates]);
+								} else {
+									module.rootDuplicates = roots[idx];
+									module.rootDuplicates.commonModules = mergeCommonModules(module.rootDuplicates.commonModules, module.getAllModuleDependencies());
+								}
+							}
+						}
+					});
+					chunk.chunks.forEach(function(chunk) {
+						if(visited.indexOf(chunk) < 0)
+							x(dups, roots, visited.concat(chunk), chunk);
+					})
+
+					currentRoots.forEach(function(roots) {
+						var commonModules = roots.commonModules;
+						var initialLength = roots.initialCcommonModulesLength;
+						if(initialLength !== commonModules.length) {
+							var template = roots[0].createTemplate(commonModules);
+							roots.template = template;
+							chunk.addModule(template);
+							template.addChunk(chunk);
+							compilation.modules.push(template);
+						}
+					});
+				}([], [], [], chunk));
+			});
+		});
+		function mergeCommonModules(commonModules, newModules) {
+			return commonModules.filter(function(module) {
+				return newModules.indexOf(module) >= 0;
+			});
+		}
+	});
+	compiler.moduleTemplate = new DedupModuleTemplateDecorator(compiler.moduleTemplate);
+	compiler.mainTemplate.renderAddModule = function(hash, chunk, varModuleId, varModule) {
+		return [
+			"var _m = " + varModule + ";",
+			"switch(typeof _m) {",
+			"case \"number\":",
+			this.indent([
+				"modules[" + varModuleId + "] = modules[_m];",
+				"break;"
+			]),
+			"case \"object\":",
+			this.indent([
+				"modules[" + varModuleId + "] = (function(_m) {",
+				this.indent([
+					"var args = _m.slice(1), fn = modules[_m[0]];",
+					"return function (a,b,c) {",
+					this.indent([
+						"fn.apply(null, [a,b,c].concat(args));"
+					]),
+					"};"
+				]),
+				"}(_m));",
+				"break;"
+			]),
+			"default:",
+			this.indent("modules[" + varModuleId + "] = _m;"),
+			"}"
+		]
+	};
+	var oldRenderModules = compiler.mainTemplate.renderModules;
+	compiler.mainTemplate.renderModules = function renderModules(hash, chunk, moduleTemplate, dependencyTemplates) {
+		var source = new ConcatSource();
+		source.add("(function(modules) {\n");
+		source.add(this.indent([
+			"for(var i in modules) {",
+			this.indent([
+				"switch(typeof modules[i]) {",
+				"case \"number\":",
+				this.indent([
+					"modules[i] = modules[modules[i]];",
+					"break;"
+				]),
+				"case \"object\":",
+				this.indent([
+					"modules[i] = (function(_m) {",
+					this.indent([
+						"var args = _m.slice(1), fn = modules[_m[0]];",
+						"return function (a,b,c) {",
+						this.indent([
+							"fn.apply(null, [a,b,c].concat(args));"
+						]),
+						"};"
+					]),
+					"}(modules[i]));"
+				]),
+				"}"
+			]),
+			"}",
+			"return modules;"
+		]));
+		source.add("\n}(");
+		source.add(oldRenderModules.call(this, hash, chunk, moduleTemplate, dependencyTemplates));
+		source.add("))");
+		return source;
+	};
+};
+
+function DedupModuleTemplateDecorator(template) {
+	this.template = template;
+}
+
+DedupModuleTemplateDecorator.prototype.render = function(module, dependencyTemplates) {
+	if(!module.rootDuplicates) return this.template.render(module, dependencyTemplates);
+	if(module.rootDuplicates.template) {
+		module.rootDuplicates.template.addReason(module, {
+			type: "template",
+			request: module.request,
+			templateModules: module.rootDuplicates.template.templateModules
+		});
+		var array = [module.rootDuplicates.template.id].concat(module.getTemplateArguments(module.rootDuplicates.template.templateModules).map(function(module) {
+			if(typeof module.id !== "number")
+				return "(function webpackMissingModule() { throw new Error(" + JSON.stringify("Cannot find module") + "); }())"
+			return module.id;
+		}));
+		var source = new ConcatSource("[" + array.join(", ") + "]");
+		return source;
+	} else {
+		module.rootDuplicates.sort(function(a, b) {
+			return a.id - b.id;
+		});
+		if(module === module.rootDuplicates[0]) return this.template.render(module, dependencyTemplates);
+		var source = new ConcatSource("" + module.rootDuplicates[0].id);
+		return source;
+	}
+};
+
+DedupModuleTemplateDecorator.prototype.updateHash = function(hash) {
+	hash.update("DedupModuleTemplateDecorator");
+	this.template.updateHash(hash);
+};

lib/webworker/WebWorkerMainTemplate.js

@@ -50,8 +50,9 @@ WebWorkerMainTemplate.prototype.renderInit = function(hash, chunk) {
 		buf.push(
 			"this[" + JSON.stringify(chunkCallbackName) + "] = function webpackChunkCallback(chunkIds, moreModules) {",
 			this.indent([
-				"for(var moduleId in moreModules)",
+				"for(var moduleId in moreModules) {",
-				this.indent("modules[moduleId] = moreModules[moduleId];"),
+				this.indent(this.renderAddModule(hash, chunk, "moduleId", "moreModules[moduleId]")),
+				"}",
 				"while(chunkIds.length)",
 				this.indent("installedChunks[chunkIds.pop()] = 1;")
 			]),

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "webpack",
-	"version": "0.10.0-beta19",
+	"version": "0.10.0-beta20",
 	"author": "Tobias Koppers @sokra",
 	"description": "Packs CommonJs/AMD/Labeled Modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loaders to preprocess files, i.e. json, jade, coffee, css, less, ... and your custom stuff.",
 	"dependencies": {

test/browsertest/build.js

@@ -34,9 +34,9 @@ var library1 = cp.spawn("node", join(["../../bin/webpack.js", "--output-pathinfo
 bindOutput(library1);
 library1.on("exit", function(code) {
 	if(code === 0) {
-		// node ../../bin/webpack --output-pathinfo --colors --resolve-alias vm=vm-browserify --output-public-path js/ --module-bind json --module-bind css=style!css --module-bind less=style!css!less --module-bind coffee --module-bind jade --prefetch ./less/stylesheet.less ./lib/index js/web.js
+		// node ../../bin/webpack --output-pathinfo --colors --resolve-alias vm=vm-browserify --output-public-path js/ --module-bind json --module-bind css=style!css --module-bind less=style!css!less --module-bind coffee --module-bind jade --prefetch ./less/stylesheet.less --optimize-dedupe ./lib/index js/web.js
 		var main = cp.spawn("node", join(["../../bin/webpack.js", "--output-pathinfo", "--colors", "--resolve-alias", "vm=vm-browserify", "--workers",
-											"--output-public-path", "js/", "--module-bind", "json", "--module-bind", "css=style!css", "--module-bind", "less=style/url!file?postfix=.css&string!less", "--module-bind", "coffee", "--module-bind", "jade", "--prefetch", "./less/stylesheet.less", "./lib/index", "js/web.js", "--progress"], extraArgs));
+											"--output-public-path", "js/", "--module-bind", "json", "--module-bind", "css=style!css", "--module-bind", "less=style/url!file?postfix=.css&string!less", "--module-bind", "coffee", "--module-bind", "jade", "--prefetch", "./less/stylesheet.less", "--optimize-dedupe", "./lib/index", "js/web.js", "--progress"], extraArgs));
 		bindOutput(main);
 	}
 });

test/browsertest/lib/dedupe1/dupdep.js

@@ -0,0 +1 @@
+module.exports = "edupe1";

test/browsertest/lib/dedupe1/index.js

@@ -0,0 +1 @@
+module.exports = require("../d") + require("./dupdep");

test/browsertest/lib/dedupe2/dupdep.js

@@ -0,0 +1 @@
+module.exports = "edupe2";

test/browsertest/lib/dedupe2/index.js

@@ -0,0 +1 @@
+module.exports = require("../d") + require("./dupdep");

test/browsertest/lib/index.web.js

@@ -550,4 +550,13 @@ describe("main", function() {
 		});
 	});
 
+	describe("deduplication", function() {
+		it("should load a duplicate module with different dependencies correctly", function() {
+			var dedupe1 = require("./dedupe1");
+			var dedupe2 = require("./dedupe2");
+			dedupe1.should.be.eql("dedupe1");
+			dedupe2.should.be.eql("dedupe2");
+		});
+	});
+
 });