opensourcemirror
/
webpack
mirror of https://github.com/webpack/webpack.git
1
0
Fork 0
Code Issues Releases Wiki Activity

cache chunks, cache assets

This commit is contained in:
Tobias Koppers 2013-05-08 13:28:54 +02:00
parent e43ec6251e
commit 5cdb8cbf81
13 changed files with 182 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bin/config-optimist.js
View File

@ -53,6 +53,8 @@ module.exports = function(optimist) {
.boolean("optimize-minimize").describe("optimize-minimize")
.boolean("optimize-occurence-order").describe("optimize-occurence-order")
.string("provide").describe("provide")
.string("plugin").describe("plugin")

6
bin/convert-argv.js
View File

@ -15,6 +15,7 @@ module.exports = function(optimist, argv, convertOptions) {
}
if(argv.p) {
argv["optimize-minimize"] = true;
argv["optimize-occurence-order"] = true;
}
function ifArg(name, fn, init) {
@ -249,6 +250,11 @@ module.exports = function(optimist, argv, convertOptions) {
options.optimize.minimize = true;
});
ifBooleanArg("optimize-occurence-order", function() {
ensureObject(options, "optimize");
options.optimize.occurenceOrder = true;
});
ifArg("provide", function(value) {
ensureObject(options, "provide");
var idx = value.indexOf("=");

6
lib/Chunk.js
View File

@ -4,11 +4,13 @@
*/
function Chunk(name) {
this.id = null;
this.ids = null;
this.name = name;
this.modules = [];
this.chunks = [];
this.parents = [];
this.blocks = [];
this.rendered = false;
}
module.exports = Chunk;
@ -127,7 +129,9 @@ Chunk.prototype.isEmpty = function() {
};
Chunk.prototype.updateHash = function(hash) {
hash.update(this.id + "");
hash.update(this.id + " ");
hash.update(this.ids ? this.ids.join(",") : "");
hash.update(this.name + "");
this.modules.forEach(function(m) {
m.updateHash(hash);
});

1
lib/ChunkTemplate.js
View File

@ -22,6 +22,7 @@ ChunkTemplate.prototype.render = function(chunk, moduleTemplate, dependencyTempl
});
source.add("\n\n");
source.add(this.asString(this.renderFooter(chunk)));
chunk.rendered = true;
return source;
};

130
lib/Compilation.js
View File

@ -46,9 +46,9 @@ Compilation.prototype = Object.create(Tapable.prototype);
Compilation.prototype.addModule = function(module) {
var identifier = module.identifier();
if(this._modules[identifier]) return false;
if(this.cache && this.cache[identifier]) {
var cacheModule = this.cache[identifier];
if(this.cache && this.cache["m" + identifier]) {
var cacheModule = this.cache["m" + identifier];
var rebuild = true;
if(!cacheModule.error && cacheModule.cacheable && this.fileTimestamps && this.contextTimestamps) {
rebuild = cacheModule.needRebuild(this.fileTimestamps, this.contextTimestamps);
@ -65,10 +65,12 @@ Compilation.prototype.addModule = function(module) {
this.warnings.push(err);
}, this);
return cacheModule;
} else {
module.lastId = cacheModule.id;
}
}
this._modules[identifier] = module;
if(this.cache) this.cache[identifier] = module;
if(this.cache) this.cache["m" + identifier] = module;
this.modules.push(module);
return true;
};
@ -164,7 +166,7 @@ Compilation.prototype.processModuleDependencies = function(module, callback) {
}
if(err) return errorOrWarningAndCallback(new ModuleNotFoundError(module, err));
if(!dependantModule) return callback();
var newModule = this.addModule(dependantModule);
if(!newModule) {
@ -180,7 +182,7 @@ Compilation.prototype.processModuleDependencies = function(module, callback) {
if(newModule instanceof Module) { // from cache
dependantModule = newModule;
dependencies.forEach(function(dep) {
dep.module = dependantModule;
dependantModule.addReason(module, dep);
@ -188,7 +190,7 @@ Compilation.prototype.processModuleDependencies = function(module, callback) {
return this.processModuleDependencies(dependantModule, callback);
}
this.buildModule(dependantModule, function(err) {
if(err) return errorOrWarningAndCallback(err);
@ -231,7 +233,7 @@ Compilation.prototype.addEntry = function process(context, entry, name, callback
if(!result) {
return callback(new Error("Entry module is already added"));
}
if(result instanceof Module) {
module = result;
}
@ -248,7 +250,7 @@ Compilation.prototype.addEntry = function process(context, entry, name, callback
entryReady.call(this);
}.bind(this));
}
function entryReady() {
this.processModuleDependencies(module, function(err) {
if(err) return callback(err);
@ -272,7 +274,9 @@ Compilation.prototype.seal = function seal(callback) {
this.applyPlugins("after-optimize-modules", this.modules);
this.applyPlugins("optimize-chunks", this.chunks);
this.applyPlugins("after-optimize-chunks", this.chunks);
this.applyPlugins("optimize-module-order", this.modules);
this.applyModuleIds();
this.applyPlugins("optimize-chunk-order", this.chunks);
this.applyChunkIds();
this.applyPlugins("optimize-module-ids", this.modules);
this.applyPlugins("after-optimize-module-ids", this.modules);
@ -337,77 +341,25 @@ Compilation.prototype.processDependenciesBlockForChunk = function processDepende
};
Compilation.prototype.applyModuleIds = function applyModuleIds() {
var i = 0;
function entryChunks(m) {
return m.chunks.filter(function(c) {
return c.entry;
}).length;
}
function occursInEntry(m) {
return m.reasons.map(function(r) {
if(!r.module) return 0;
return entryChunks(r.module);
}).reduce(function(a, b) { return a+b; }, 0) + entryChunks(m);
}
function occurs(m) {
return m.reasons.map(function(r) {
if(!r.module) return 0;
return r.module.chunks.length;
}).reduce(function(a, b) { return a+b; }, 0) + m.chunks.length;
}
this.modules.sort(function(a, b) {
var aEntryOccurs = occursInEntry(a);
var bEntryOccurs = occursInEntry(b);
if(aEntryOccurs > bEntryOccurs) return -1;
if(aEntryOccurs < bEntryOccurs) return 1;
var aOccurs = occurs(a);
var bOccurs = occurs(b);
if(aOccurs > bOccurs) return -1;
if(aOccurs < bOccurs) return 1;
if(a.identifier() > b.identifier()) return 1;
if(a.identifier() < b.identifier()) return -1;
return 0;
});
var i = this.cache && this.cache["nextModuleId"] || 1;
var usedIds = {0:true};
this.modules.forEach(function(module) {
if(module.id === null)
module.id = ++i;
if(module.id === null) {
if(module.lastId > 0) {
if(!usedIds[module.lastId]) {
usedIds[module.lastId] = true;
module.id = module.lastId;
return;
}
}
module.id = i++;
}
});
if(this.cache) this.cache["nextModuleId"] = i;
};
Compilation.prototype.applyChunkIds = function applyChunkIds() {
var i = 0;
function occursInEntry(c) {
return c.parents.filter(function(p) {
return p.entry;
}).length;
}
function occurs(c) {
return c.blocks.length;
}
this.chunks.forEach(function(c) {
c.modules.sort(function(a, b) {
if(a.identifier() > b.identifier()) return 1;
if(a.identifier() < b.identifier()) return -1;
return 0;
});
});
this.chunks.sort(function(a, b) {
var aEntryOccurs = occursInEntry(a);
var bEntryOccurs = occursInEntry(b);
if(aEntryOccurs > bEntryOccurs) return -1;
if(aEntryOccurs < bEntryOccurs) return 1;
var aOccurs = occurs(a);
var bOccurs = occurs(b);
if(aOccurs > bOccurs) return -1;
if(aOccurs < bOccurs) return 1;
if(a.modules.length > b.modules.length) return -1;
if(a.modules.length < b.modules.length) return 1;
for(var i = 0; i < a.modules.length; i++) {
if(a.modules[i].identifier() > b.modules[i].identifier()) return -1;
if(a.modules[i].identifier() < b.modules[i].identifier()) return 1;
}
return 0;
});
this.chunks.forEach(function(chunk) {
if(chunk.id === null)
chunk.id = ++i;
@ -494,7 +446,17 @@ Compilation.prototype.createChunkAssets = function createChunkAssets() {
var source;
var file;
if(chunk.entry) {
source = this.mainTemplate.render(hash, chunk, this.moduleTemplate, this.dependencyTemplates);
if(this.cache && this.cache["c" + chunk.id + chunk.name] && this.cache["c" + chunk.id + chunk.name].hash == hash) {
source = this.cache["c" + chunk.id + chunk.name].source;
} else {
source = this.mainTemplate.render(hash, chunk, this.moduleTemplate, this.dependencyTemplates);
if(this.cache) {
this.cache["c" + chunk.id + chunk.name] = {
hash: hash,
source: source
}
}
}
this.assets[
file = filename
.replace(Template.REGEXP_HASH, hash)
@ -504,7 +466,23 @@ Compilation.prototype.createChunkAssets = function createChunkAssets() {
chunk.files.push(file);
this.applyPlugins("chunk-asset", chunk, file);
} else {
source = this.chunkTemplate.render(chunk, this.moduleTemplate, this.dependencyTemplates);
if(this.cache) {
var chunkHash = new (require("crypto").Hash)("md5");
chunk.updateHash(chunkHash);
this.chunkTemplate.updateHash(chunkHash);
chunkHash = chunkHash.digest("hex");
}
if(this.cache && this.cache["c" + chunk.id] && this.cache["c" + chunk.id].hash == chunkHash) {
source = this.cache["c" + chunk.id].source;
} else {
source = this.chunkTemplate.render(chunk, this.moduleTemplate, this.dependencyTemplates);
if(this.cache) {
this.cache["c" + chunk.id] = {
hash: chunkHash,
source: source
}
}
}
this.assets[
file = chunkFilename
.replace(Template.REGEXP_HASH, hash)

14
lib/Compiler.js
View File

@ -100,7 +100,7 @@ function Compiler() {
this.outputFileSystem = null;
this.inputFileSystem = null;
this.separateExecutor = null;
this.fileTimestamps = {};
this.contextTimestamps = {};
@ -179,10 +179,18 @@ Compiler.prototype.emitAssets = function(compilation, callback) {
} else writeOut.call(this);
function writeOut(err) {
if(err) return callback(err);
var content = compilation.assets[file].source();
var targetPath = this.outputFileSystem.join(this.outputPath, file);
var source = compilation.assets[file];
if(source.existsAt === targetPath) {
source.emitted = false;
return callback();
}
var content = source.source();
if(!Buffer.isBuffer(content))
content = new Buffer(content, "utf-8");
this.outputFileSystem.writeFile(this.outputFileSystem.join(this.outputPath, file), content, callback);
source.existsAt = targetPath;
source.emitted = true;
this.outputFileSystem.writeFile(targetPath, content, callback);
};
}.bind(this), function(err) {

1
lib/MainTemplate.js
View File

@ -42,6 +42,7 @@ MainTemplate.prototype.render = function(hash, chunk, moduleTemplate, dependency
source.add(moduleTemplate.render(module, dependencyTemplates));
});
source.add("\n/******/ })");
chunk.rendered = true;
return source;
};

2
lib/Module.js
View File

@ -11,6 +11,7 @@ function Module() {
this.context = null;
this.reasons = [];
this.debugId = debugId++;
this.lastId = -1;
this.id = null;
this.chunks = [];
this.warnings = [];
@ -26,6 +27,7 @@ Module.prototype.separable = function(callback) {
Module.prototype.disconnect = function() {
this.reasons.length = 0;
this.lastId = this.id;
this.id = null;
this.chunks.length = 0;
DependenciesBlock.prototype.disconnect.call(this);

17
lib/Stats.js
View File

@ -87,7 +87,8 @@ Stats.prototype.toJson = function toJson(options, forToString) {
name: asset,
size: compilation.assets[asset].size(),
chunks: [],
chunkNames: []
chunkNames: [],
emitted: compilation.assets[asset].emitted
};
assetsByFile[asset] = obj;
return obj;
@ -142,6 +143,7 @@ Stats.prototype.toJson = function toJson(options, forToString) {
obj.chunks = compilation.chunks.map(function(chunk) {
var obj = {
id: chunk.id,
rendered: chunk.rendered,
size: chunk.modules.reduce(function(size, module) { return size + module.size(); }, 0),
names: chunk.name ? [chunk.name] : [],
files: chunk.files.slice(),
@ -277,16 +279,17 @@ Stats.jsonToString = function jsonToString(obj, useColors) {
newline();
}
if(obj.assets) {
var t = [["Asset", "Size", "Chunks", "Chunk Names"]]
var t = [["Asset", "Size", "Chunks", "", "Chunk Names"]]
obj.assets.forEach(function(asset) {
t.push([
asset.name,
asset.size,
asset.chunks.join(", "),
asset.emitted ? "[emitted]" : "",
asset.chunkNames.join(", ")
])
});
table(t, [green, normal, bold, normal], "rrrl");
table(t, [green, normal, bold, green, normal], "rrrll");
}
if(obj.chunks) {
obj.chunks.forEach(function(chunk) {
@ -306,11 +309,13 @@ Stats.jsonToString = function jsonToString(obj, useColors) {
normal(" ");
normal(chunk.size);
chunk.parents.forEach(function(id) {
normal(" ");
normal("{");
normal(" {");
yellow(id);
normal("} ");
normal("}");
});
if(chunk.rendered) {
green(" [rendered]");
}
newline();
if(chunk.modules) {
chunk.modules.forEach(function(module) {

5
lib/WebpackOptionsApply.js
View File

@ -34,6 +34,7 @@ var RequireContextPlugin = require("./dependencies/RequireContextPlugin");
var RequireEnsurePlugin = require("./dependencies/RequireEnsurePlugin");
var RequireIncludePlugin = require("./dependencies/RequireIncludePlugin");
var OccurenceOrderPlugin = require("./optimize/OccurenceOrderPlugin");
var LimitChunkCountPlugin = require("./optimize/LimitChunkCountPlugin");
var MinChunkSizePlugin = require("./optimize/MinChunkSizePlugin");
var RemoveParentModulesPlugin = require("./optimize/RemoveParentModulesPlugin");
@ -130,6 +131,10 @@ WebpackOptionsApply.prototype.process = function(options, compiler) {
new MergeDuplicateChunksPlugin(),
new FlagIncludedChunksPlugin()
);
if(options.optimize && options.optimize.occurenceOrder)
compiler.apply(new OccurenceOrderPlugin(options.optimize.occurenceOrderPreferEntry));
if(options.optimize && options.optimize.minChunkSize)
compiler.apply(new MinChunkSizePlugin(options.optimize.minChunkSize));

2
lib/WebpackOptionsDefaulter.js
View File

@ -41,6 +41,8 @@ function WebpackOptionsDefaulter() {
this.set("resolve.packageMains", ["webpack", "browser", "web", "main"]);
this.set("resolveLoader.packageMains", ["webpackLoader", "webLoader", "loader", "main"]);
this.set("optimize.occurenceOrderPreferEntry", true);
}
module.exports = WebpackOptionsDefaulter;

81
lib/optimize/OccurenceOrderPlugin.js Normal file
View File

@ -0,0 +1,81 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
function OccurenceOrderPlugin(preferEntry) {
this.preferEntry = preferEntry;
}
module.exports = OccurenceOrderPlugin;
OccurenceOrderPlugin.prototype.apply = function(compiler) {
var preferEntry = this.preferEntry;
compiler.plugin("compilation", function(compilation) {
compilation.plugin("optimize-module-order", function(modules) {
function entryChunks(m) {
return m.chunks.filter(function(c) {
return c.entry;
}).length;
}
function occursInEntry(m) {
return m.reasons.map(function(r) {
if(!r.module) return 0;
return entryChunks(r.module);
}).reduce(function(a, b) { return a+b; }, 0) + entryChunks(m);
}
function occurs(m) {
return m.reasons.map(function(r) {
if(!r.module) return 0;
return r.module.chunks.length;
}).reduce(function(a, b) { return a+b; }, 0) + m.chunks.length;
}
modules.sort(function(a, b) {
if(preferEntry) {
var aEntryOccurs = occursInEntry(a);
var bEntryOccurs = occursInEntry(b);
if(aEntryOccurs > bEntryOccurs) return -1;
if(aEntryOccurs < bEntryOccurs) return 1;
}
var aOccurs = occurs(a);
var bOccurs = occurs(b);
if(aOccurs > bOccurs) return -1;
if(aOccurs < bOccurs) return 1;
if(a.identifier() > b.identifier()) return 1;
if(a.identifier() < b.identifier()) return -1;
return 0;
});
});
compilation.plugin("optimize-chunk-order", function(chunks) {
function occursInEntry(c) {
return c.parents.filter(function(p) {
return p.entry;
}).length;
}
function occurs(c) {
return c.blocks.length;
}
chunks.forEach(function(c) {
c.modules.sort(function(a, b) {
if(a.identifier() > b.identifier()) return 1;
if(a.identifier() < b.identifier()) return -1;
return 0;
});
});
chunks.sort(function(a, b) {
var aEntryOccurs = occursInEntry(a);
var bEntryOccurs = occursInEntry(b);
if(aEntryOccurs > bEntryOccurs) return -1;
if(aEntryOccurs < bEntryOccurs) return 1;
var aOccurs = occurs(a);
var bOccurs = occurs(b);
if(aOccurs > bOccurs) return -1;
if(aOccurs < bOccurs) return 1;
if(a.modules.length > b.modules.length) return -1;
if(a.modules.length < b.modules.length) return 1;
for(var i = 0; i < a.modules.length; i++) {
if(a.modules[i].identifier() > b.modules[i].identifier()) return -1;
if(a.modules[i].identifier() < b.modules[i].identifier()) return 1;
}
return 0;
});
});
});
};

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "webpack",
"version": "0.10.0-beta5",
"version": "0.10.0-beta6",
"author": "Tobias Koppers @sokra",
"description": "Packs CommonJs/AMD/Labeled Modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loaders to preprocess files, i.e. json, jade, coffee, css, less, ... and your custom stuff.",
"dependencies": {