opensourcemirror
/
webpack
mirror of https://github.com/webpack/webpack.git
1
0
Fork 0
Code Issues Releases Wiki Activity

seperate resolving

This commit is contained in:
Tobias Koppers 2012-09-26 12:28:23 +02:00
parent 0b9342e863
commit 18db563ef6
7 changed files with 111 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
README.md
View File

@ -497,6 +497,8 @@ You can also save this options object in a JSON file and use it with the shell c
// defaults: []
// postprocess resolved filenames by all specified async functions
// a postprocessor must call the callback
// You must pass a filename instead of a function if you use workers
// The filename is required in the worker process.
context: [],
// same as postprocess.normal but for contextes
@ -508,12 +510,14 @@ You can also save this options object in a JSON file and use it with the shell c
// syntax like resolve.loaders
// all loaders which matches the file are applied after the
// normal loaders. This cannot be overridden in the require call.
// You must pass a string instead of a RegExp if you use workers
preLoaders: [{test: /\.txt$|\.html$/, loader: "normalizeNLs"}],
// default: []
// syntax like resolve.loaders
// all loaders which matches the file are applied before the
// normal loaders. This cannot be overridden in the require call.
// You must pass a string instead of a RegExp if you use workers
workers: true,
// default: false
@ -529,6 +533,11 @@ You can also save this options object in a JSON file and use it with the shell c
// default: true
// close the worker processes on webpack exit.
workersNoResolve: true,
// default: false
// workers should not be used in the resolving process
// This may be useful if you want to have your postprocessors in the main process.
profile: true,
// default: false
// capture timings for the build.

46
lib/Workers.js
View File

@ -3,19 +3,19 @@ var child_process = require("child_process");
function Workers(moduleToFork, count) {
this.nextId = 1;
this.workers = [];
this.forkedWorkers = [];
this.workersJobs = [];
this.callbacks = {};
for(var i = 0; i < count; i++) {
var worker = child_process.fork(moduleToFork);
this.workers.push(worker);
this.workersJobs.push(0);
this.bindWorker(worker, i);
}
}
module.exports = Workers;
Workers.prototype.run = function(parameters, callback) {
var worker = 0;
Workers.prototype.run = function() {
var args = Array.prototype.slice.call(arguments, 0);
var worker = -1;
var minJobs = -1;
this.workersJobs.forEach(function(jobs, idx) {
if(jobs < minJobs || minJobs == -1) {
@ -23,24 +23,44 @@ Workers.prototype.run = function(parameters, callback) {
worker = idx;
}
});
if(worker < 0) throw new Error("Not ready! Check workers.ready().");
this.workersJobs[worker]++;
this.pushJob(this.workers[worker], parameters, callback);
args.unshift(this.workers[worker]);
this.pushJob.apply(this, args);
}
Workers.prototype.bindWorker = function(worker, idx) {
worker.on("message", function(result) {
this.workersJobs[idx]--;
var id = result.shift();
var callback = this.callbacks[id];
delete this.callbacks[id];
callback.apply(null, result);
worker.send(null);
worker.once("message", function(msg) {
if(msg == "ready") {
idx = this.workers.length;
this.workers.push(worker);
this.workersJobs.push(0);
worker.on("message", function(result) {
this.workersJobs[idx]--;
var id = result.shift();
var callback = this.callbacks[id];
delete this.callbacks[id];
callback.apply(null, result);
}.bind(this));
} else {
worker.disconnect();
this.forkedWorkers.splice(idx, 1);
}
}.bind(this));
}
Workers.prototype.pushJob = function(worker, parameters, callback) {
Workers.prototype.ready = function() {
return this.workers.length > 0;
}
Workers.prototype.pushJob = function(worker) {
var args = Array.prototype.slice.call(arguments, 0);
var callback = args.pop();
var id = this.nextId++;
this.callbacks[id] = callback;
worker.send([id, parameters]);
args[0] = id;
worker.send(args);
}
Workers.prototype.close = function() {

31
lib/buildDeps.js
View File

@ -125,7 +125,8 @@ function addModule(depTree, context, modu, options, reason, finalCallback) {
}
// resolve the filename of the required module
resolve(context = context || path.dirname(modu), modu, options.resolve, resolved);
var resolveFunc = !options.workersNoResolve && options.workers && options.workers.ready() ? options.workers.run.bind(options.workers, "resolve") : resolve;
resolveFunc(context = context || path.dirname(modu), modu, options.resolve, resolved);
function resolved(err, filename) {
if(err) {
callback(err);
@ -174,21 +175,28 @@ function addModule(depTree, context, modu, options, reason, finalCallback) {
var preLoaders = options.preLoaders ? matchLoadersList(options.preLoaders) : "";
var postLoaders = options.postLoaders ? matchLoadersList(options.postLoaders) : "";
resolve.loaders(context, preLoaders, options.resolve, function(err, preLoaders) {
var resolveLoadersFunc = !options.workersNoResolve && options.workers && options.workers.ready() ?
options.workers.run.bind(options.workers, "resolve.loaders") :
resolve.loaders;
if(preLoaders) resolveLoadersFunc(context, preLoaders, options.resolve, onPreLoadersResolved);
else onPreLoadersResolved(null, []);
function onPreLoadersResolved(err, preLoaders) {
if(err) return callback(err);
resolve.loaders(context, postLoaders, options.resolve, function(err, postLoaders) {
resolveLoadersFunc(context, postLoaders, options.resolve, onPostLoadersResolved);
function onPostLoadersResolved(err, postLoaders) {
if(err) return callback(err);
profile && (profile.resolvePrePostLoadersEnd = new Date());
var allLoaders = [];
allLoaders.push.apply(allLoaders, preLoaders);
allLoaders.push.apply(allLoaders, loaders);
allLoaders.push.apply(allLoaders, postLoaders);
var seperate = !!options.workers;
var seperate = !!(options.workers && options.workers.ready());
try {
for(var i = 0; i < allLoaders.length && seperate; i++) {
var loaderFilename = allLoaders[i];
options.events && options.events.emit("loader", loaderFilename);
if(!require(loaderFilename).seperable)
var loader = require(loaderFilename);
if(!loader.seperable && (!loader.seperableIfResolve || options.workersNoResolve))
seperate = false;
}
} catch(e) {
@ -225,8 +233,8 @@ function addModule(depTree, context, modu, options, reason, finalCallback) {
}
return processParsedJs(source, deps);
});
});
});
}
}
}
function matchLoadersList(list) {
@ -242,11 +250,10 @@ function addModule(depTree, context, modu, options, reason, finalCallback) {
Object.keys(parameters.options).forEach(function(name) {
if(name == "internal") return;
if(name == "events") return;
if(name == "resolve") return;
opt[name] = parameters.options[name];
});
parameters.options = opt;
options.workers.run(parameters, function(err, source, deps, cacheInfo, profileBuild) {
options.workers.run("buildModule", parameters, function(err, source, deps, cacheInfo, profileBuild) {
if(err) err = {
message: err.message,
stack: err.stack,
@ -449,7 +456,9 @@ function addContextModule(depTree, context, contextModuleName, options, reason,
match = true;
if(options.resolve && options.resolve.loaders)
options.resolve.loaders.forEach(function(loader) {
if(loader.test.test(filename))
var test = loader.test;
if(typeof test == "string") test = new RegExp(test);
if(test.test(filename))
match = true;
});
});

14
lib/buildModuleFork.js
View File

@ -1,14 +0,0 @@
var buildModule = require("./buildModule");
process.on("message", function(arr) {
var id = arr[0]
var parameters = arr[1];
try {
buildModule(parameters, function(err, source, deps, cacheInfo, profile) {
if(err) err = { message: err.message, stack: err.stack, _toString: err.toString() };
process.send([id, err, source, deps, cacheInfo, profile]);
});
} catch(e) {
process.send([id, e]);
}
});

12
lib/webpack.js
View File

@ -52,7 +52,7 @@ module.exports = function webpackMain(context, moduleName, options, callback) {
options.workers = require("os").cpus().length;
}
if(typeof options.workers == "number") {
options.workers = new (require("./Workers"))(path.join(__dirname, "buildModuleFork.js"), options.workers);
options.workers = new (require("./Workers"))(path.join(__dirname, "worker.js"), options.workers);
}
if(options.workers && options.closeWorkers !== false) {
if(options.watch) {
@ -98,11 +98,11 @@ module.exports = function webpackMain(context, moduleName, options, callback) {
options.resolve.paths.push(path.join(path.dirname(__dirname), "node_modules"));
options.resolve.alias = options.resolve.alias || {};
options.resolve.loaders = options.resolve.loaders || [];
options.resolve.loaders.push({test: /\.coffee$/, loader: "coffee"});
options.resolve.loaders.push({test: /\.json$/, loader: "json"});
options.resolve.loaders.push({test: /\.jade$/, loader: "jade"});
options.resolve.loaders.push({test: /\.css$/, loader: "style!css"});
options.resolve.loaders.push({test: /\.less$/, loader: "style!css!val/cacheable!less"});
options.resolve.loaders.push({test: "\\.coffee$", loader: "coffee"});
options.resolve.loaders.push({test: "\\.json$", loader: "json"});
options.resolve.loaders.push({test: "\\.jade$", loader: "jade"});
options.resolve.loaders.push({test: "\\.css$", loader: "style!css"});
options.resolve.loaders.push({test: "\\.less$", loader: "style!css!val/cacheable!less"});
if(!options.resolve.extensions)
options.resolve.extensions = ["", ".webpack.js", ".web.js", ".js"];
if(!options.resolve.postfixes)

42
lib/worker.js Normal file
View File

@ -0,0 +1,42 @@
var buildModule = require("./buildModule");
var resolve = require("enhanced-resolve");
process.on("message", function(msg) {
if(!msg) return process.send("ready");
// setTimeout(function() {
var id = msg[0];
try {
switch(msg[1]) {
case "buildModule":
var parameters = msg[2];
buildModule(parameters, function(err, source, deps, cacheInfo, profile) {
if(err) err = { message: err.message, stack: err.stack, _toString: err.toString() };
process.send([id, err, source, deps, cacheInfo, profile]);
});
break;
case "resolve":
var context = msg[2];
var module = msg[3];
var options = msg[4];
resolve(context, module, options, function(err, filename) {
if(err) err = { message: err.message, stack: err.stack, _toString: err.toString() };
process.send([id, err, filename]);
})
break;
case "resolve.loaders":
var context = msg[2];
var loaders = msg[3];
var options = msg[4];
resolve.loaders(context, loaders, options, function(err, loaders) {
if(err) err = { message: err.message, stack: err.stack, _toString: err.toString() };
process.send([id, err, loaders]);
})
break;
default: throw new Error("Worker msg type " + msg[1] + " is not supported");
}
} catch(e) {
process.send([id, e]);
}
// }, 2000);
});

2
package.json
View File

@ -1,6 +1,6 @@
{
"name": "webpack",
"version": "0.7.0-beta3",
"version": "0.7.0-beta4",
"author": "Tobias Koppers @sokra",
"description": "Packs CommonJs/AMD Modules for the browser. Allows to split your codebase into multiple bundles, which can be loaded on demand. Support loading of js, json, jade, coffee, css, ... out of the box and more with custom loaders.",
"dependencies": {