opensourcemirror
/
webpack
mirror of https://github.com/webpack/webpack.git
1
0
Fork 0
Code Issues Releases Wiki Activity

better work distribution

This commit is contained in:
Tobias Koppers 2012-09-25 19:23:05 +02:00
parent 193173c7b4
commit 0b9342e863
8 changed files with 49 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@ -523,8 +523,7 @@ You can also save this options object in a JSON file and use it with the shell c
// overhead (~100-200ms). If loaders are used they need to have the
// seperable flag to work in worker process. If they havn't they work in
// the main process.
// In watch mode, worker processes only start once. So workers = true
// is recommended for watch mode.
// Pushing jobs to worker processes has an addititional overhead of ~100ms.
closeWorkers: false,
// default: true

2
bm.js
View File

@ -5,7 +5,7 @@
var webpack = require("./lib/webpack");
var path = require("path");
var TIMES = 2;
var TIMES = 5;
/* TESTS */

41
lib/Workers.js
View File

@ -2,41 +2,38 @@ var child_process = require("child_process");
function Workers(moduleToFork, count) {
this.nextId = 1;
this.jobs = [];
this.freeWorkers = [];
this.workers = [];
this.workersJobs = [];
this.callbacks = {};
for(var i = 0; i < count; i++) {
var worker = child_process.fork(moduleToFork);
this.workers.push(worker);
this.freeWorkers.push(worker);
this.bindWorker(worker);
this.workersJobs.push(0);
this.bindWorker(worker, i);
}
}
module.exports = Workers;
Workers.prototype.run = function(parameters, callback) {
if(this.freeWorkers.length > 0)
this.pushJob(this.freeWorkers.shift(), parameters, callback);
else
this.jobs.push([parameters, callback]);
var worker = 0;
var minJobs = -1;
this.workersJobs.forEach(function(jobs, idx) {
if(jobs < minJobs || minJobs == -1) {
minJobs = jobs;
worker = idx;
}
});
this.workersJobs[worker]++;
this.pushJob(this.workers[worker], parameters, callback);
}
Workers.prototype.bindWorker = function(worker) {
Workers.prototype.bindWorker = function(worker, idx) {
worker.on("message", function(result) {
if(Array.isArray(result)) {
var id = result.shift();
var callback = this.callbacks[id];
delete this.callbacks[id];
callback.apply(null, result);
} else {
if(this.jobs.length > 0) {
var job = this.jobs.shift();
this.pushJob(worker, job[0], job[1]);
} else {
this.freeWorkers.push(worker);
}
}
this.workersJobs[idx]--;
var id = result.shift();
var callback = this.callbacks[id];
delete this.callbacks[id];
callback.apply(null, result);
}.bind(this));
}

22
lib/buildDeps.js
View File

@ -104,13 +104,21 @@ function addModule(depTree, context, modu, options, reason, finalCallback) {
options.events.emit("task-end");
if(profile && profile.module) {
profile.end = new Date();
profile.module.profile = {
time: profile.end - profile.start,
timeResolve: profile.resolveEnd - profile.start,
timeResolvePrePostLoaders: profile.resolvePrePostLoadersEnd - profile.resolveEnd,
timeLoadersCheck: profile.loadersCheckEnd - profile.resolvePrePostLoadersEnd,
timeBuildModule: profile.buildModuleEnd - profile.loadersCheckEnd,
timeChildren: profile.end - profile.buildModuleEnd
if(profile.buildModule) {
profile.module.profile = {
time: profile.end - profile.start,
timeResolve: profile.resolveEnd - profile.start,
timeResolvePrePostLoaders: profile.resolvePrePostLoadersEnd - profile.resolveEnd,
timeLoadersCheck: profile.loadersCheckEnd - profile.resolvePrePostLoadersEnd,
timeBuildWaiting: (profile.buildModuleEnd - profile.loadersCheckEnd) - (profile.buildModule.end - profile.buildModule.start),
timeBuildModule: profile.buildModule.end - profile.buildModule.start,
timeBuildModuleRead: profile.buildModule.readEnd - profile.buildModule.start,
timeBuildModulePreLoaders: profile.buildModule.preLoadersEnd - profile.buildModule.readEnd,
timeBuildModuleLoaders: profile.buildModule.loadersEnd - profile.buildModule.preLoadersEnd,
timeBuildModulePostLoaders: profile.buildModule.postLoadersEnd - profile.buildModule.loadersEnd,
timeBuildModuleParse: profile.buildModule.end - profile.buildModule.postLoadersEnd,
timeChildren: profile.end - profile.buildModuleEnd
}
}
}
finalCallback(err, result);

12
lib/buildModule.js
View File

@ -12,6 +12,10 @@ function buildModule(parameters, callback) {
var filename = parameters.filename;
var options = parameters.options;
if(options.profile) var profile = {
start: new Date().getTime()
}
var dependencyInfo = {
cacheable: true,
files: [filename]
@ -28,6 +32,8 @@ function buildModule(parameters, callback) {
fs.readFile(filename, function(err, content) {
if(err) return callback(err);
profile && (profile.readEnd = new Date().getTime());
var loaderContext = {
loaders: loaders,
preLoaders: preLoaders,
@ -39,14 +45,17 @@ function buildModule(parameters, callback) {
execLoaders(context, filenameWithLoaders, preLoaders, [filename], [content], loaderContext, dependencyInfo, options,
function(err, result) {
if(err) return callback(err);
profile && (profile.preLoadersEnd = new Date().getTime());
loaderContext.loaderType = "loader";
execLoaders(context, filenameWithLoaders, loaders, [filename], result, loaderContext, dependencyInfo, options,
function(err, result) {
if(err) return callback(err);
profile && (profile.loadersEnd = new Date().getTime());
loaderContext.loaderType = "postLoader";
execLoaders(context, filenameWithLoaders, postLoaders, [filename], result, loaderContext, dependencyInfo, options,
function(err, result) {
if(err) return callback(err);
profile && (profile.postLoadersEnd = new Date().getTime());
return processJs(result)
});
});
@ -66,7 +75,8 @@ function buildModule(parameters, callback) {
callback(new Error("File \"" + filenameWithLoaders + "\" parsing failed: " + e));
return;
}
return callback(null, source, deps, dependencyInfo);
profile && (profile.end = new Date().getTime());
return callback(null, source, deps, dependencyInfo, profile);
}
}
module.exports = buildModule;

1
lib/buildModuleFork.js
View File

@ -11,5 +11,4 @@ process.on("message", function(arr) {
} catch(e) {
process.send([id, e]);
}
process.send(null);
});

7
lib/formatOutput.js
View File

@ -71,12 +71,7 @@ module.exports = function(stats, options) {
var valueTime = module.profile.time - module.profile.timeChildren;
if(valueTime > maxTime) moduleLine += c("\033[1m\033[31m");
else if(valueTime > middleTime) moduleLine += c("\033[1m\033[33m");
var buildProfile = module.profile.buildModule;
if(buildProfile) buildProfile = " (" +
(buildProfile.loaders && buildProfile.loaders.length > 0 ? "loaders: " + buildProfile.loaders.join("ms, ") + "ms" : "") +
")";
else buildProfile = "";
moduleLine += " [" + module.profile.time + "ms: " + (module.profile.timeResolve + module.profile.timeResolvePrePostLoaders) + "ms resolving, " + module.profile.timeBuildModule + "ms build" + buildProfile + ", " + module.profile.timeChildren + "ms children]";
moduleLine += " [" + module.profile.time + "ms: " + (module.profile.timeResolve + module.profile.timeResolvePrePostLoaders) + "ms resolving, " + module.profile.timeBuildWaiting + "ms waiting" + ", " + module.profile.timeBuildModule + "ms build" + ", " + module.profile.timeChildren + "ms children]";
if(valueTime > middleTime) moduleLine += c("\033[39m\033[22m");
}
buf.push(moduleLine);

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "webpack",
"version": "0.7.0-beta2",
"version": "0.7.0-beta3",
"author": "Tobias Koppers @sokra",
"description": "Packs CommonJs/AMD Modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loading of js, json, jade, coffee, css, ... out of the box and more with custom loaders.",
"dependencies": {