opensourcemirror
/
webpack
mirror of https://github.com/webpack/webpack.git
1
0
Fork 0
Code Issues Releases Wiki Activity

refactor Cache 

add a new hookable Cache class
removed CachePlugin
add MemoryCachePlugin
refactor timestamps
create FileSystemInfo class
This commit is contained in:
Tobias Koppers 2018-09-27 07:22:19 +02:00
parent f31a8c231b
commit 7340fbb547
23 changed files with 623 additions and 468 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
lib/Cache.js Normal file
View File

@ -0,0 +1,62 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
const { AsyncParallelHook, AsyncSeriesBailHook, SyncHook } = require("tapable");
/** @typedef {import("webpack-sources").Source} Source */
/** @typedef {import("./Module")} Module */
class Cache {
constructor() {
this.hooks = {
/** @type {AsyncSeriesBailHook<string>} */
getModule: new AsyncSeriesBailHook(["identifier"]),
/** @type {AsyncParallelHook<string, Module>} */
storeModule: new AsyncParallelHook(["identifier", "module"]),
/** @type {AsyncSeriesBailHook<string, string>} */
getAsset: new AsyncSeriesBailHook(["identifier", "hash"]),
/** @type {AsyncParallelHook<string, string, Source>} */
storeAsset: new AsyncParallelHook(["identifier", "hash", "source"]),
/** @type {SyncHook} */
beginIdle: new SyncHook([]),
/** @type {AsyncParallelHook} */
endIdle: new AsyncParallelHook([]),
/** @type {AsyncParallelHook} */
shutdown: new AsyncParallelHook([])
};
}
getModule(identifier, callback) {
this.hooks.getModule.callAsync(identifier, callback);
}
storeModule(identifier, module, callback) {
this.hooks.storeModule.callAsync(identifier, module, callback);
}
getAsset(identifier, hash, callback) {
this.hooks.getAsset.callAsync(identifier, hash, callback);
}
storeAsset(identifier, hash, source, callback) {
this.hooks.storeAsset.callAsync(identifier, hash, source, callback);
}
beginIdle() {
this.hooks.beginIdle.call();
}
endIdle(callback) {
this.hooks.endIdle.callAsync(callback);
}
shutdown(callback) {
this.hooks.shutdown.callAsync(callback);
}
}
module.exports = Cache;

103
lib/CachePlugin.js
View File

@ -1,103 +0,0 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
const asyncLib = require("neo-async");
class CachePlugin {
constructor(cache) {
this.cache = cache || {};
this.FS_ACCURACY = 2000;
}
apply(compiler) {
if (Array.isArray(compiler.compilers)) {
compiler.compilers.forEach((c, idx) => {
new CachePlugin((this.cache[idx] = this.cache[idx] || {})).apply(c);
});
} else {
const registerCacheToCompiler = (compiler, cache) => {
compiler.hooks.thisCompilation.tap("CachePlugin", compilation => {
compilation.cache = cache;
compilation.hooks.childCompiler.tap(
"CachePlugin",
(childCompiler, compilerName, compilerIndex) => {
if (cache) {
let childCache;
if (!cache.children) {
cache.children = {};
}
if (!cache.children[compilerName]) {
cache.children[compilerName] = [];
}
if (cache.children[compilerName][compilerIndex]) {
childCache = cache.children[compilerName][compilerIndex];
} else {
cache.children[compilerName].push((childCache = {}));
}
registerCacheToCompiler(childCompiler, childCache);
}
}
);
});
};
registerCacheToCompiler(compiler, this.cache);
compiler.hooks.watchRun.tap("CachePlugin", () => {
this.watching = true;
});
compiler.hooks.run.tapAsync("CachePlugin", (compiler, callback) => {
if (!compiler._lastCompilationFileDependencies) {
return callback();
}
const fs = compiler.inputFileSystem;
const fileTs = (compiler.fileTimestamps = new Map());
asyncLib.forEach(
compiler._lastCompilationFileDependencies,
(file, callback) => {
fs.stat(file, (err, stat) => {
if (err) {
if (err.code === "ENOENT") return callback();
return callback(err);
}
if (stat.mtime) this.applyMtime(+stat.mtime);
fileTs.set(file, +stat.mtime || Infinity);
callback();
});
},
err => {
if (err) return callback(err);
for (const [file, ts] of fileTs) {
fileTs.set(file, ts + this.FS_ACCURACY);
}
callback();
}
);
});
compiler.hooks.afterCompile.tap("CachePlugin", compilation => {
compilation.compiler._lastCompilationFileDependencies =
compilation.fileDependencies;
compilation.compiler._lastCompilationContextDependencies =
compilation.contextDependencies;
});
}
}
/* istanbul ignore next */
applyMtime(mtime) {
if (this.FS_ACCURACY > 1 && mtime % 2 !== 0) this.FS_ACCURACY = 1;
else if (this.FS_ACCURACY > 10 && mtime % 20 !== 0) this.FS_ACCURACY = 10;
else if (this.FS_ACCURACY > 100 && mtime % 200 !== 0)
this.FS_ACCURACY = 100;
else if (this.FS_ACCURACY > 1000 && mtime % 2000 !== 0)
this.FS_ACCURACY = 1000;
}
}
module.exports = CachePlugin;

311
lib/Compilation.js
View File

@ -21,6 +21,7 @@ const ChunkRenderError = require("./ChunkRenderError");
const ChunkTemplate = require("./ChunkTemplate");
const DependencyTemplates = require("./DependencyTemplates");
const Entrypoint = require("./Entrypoint");
const FileSystemInfo = require("./FileSystemInfo");
const {
connectChunkGroupAndChunk,
connectChunkGroupParentAndChild
@ -55,6 +56,7 @@ const { arrayToSetDeprecation } = require("./util/deprecation");
/** @typedef {import("./Dependency").DependencyLocation} DependencyLocation */
/** @typedef {import("./DependencyTemplate")} DependencyTemplate */
/** @typedef {import("./Module")} Module */
/** @typedef {import("./Template").RenderManifestEntry} RenderManifestEntry */
/** @typedef {import("./WebpackError")} WebpackError */
/** @typedef {import("./dependencies/DependencyReference")} DependencyReference */
/** @typedef {import("./dependencies/DllEntryDependency")} DllEntryDependency */
@ -345,8 +347,10 @@ class Compilation {
this.compiler = compiler;
this.resolverFactory = compiler.resolverFactory;
this.inputFileSystem = compiler.inputFileSystem;
this.fileSystemInfo = new FileSystemInfo(this.inputFileSystem);
this.requestShortener = compiler.requestShortener;
this.compilerPath = compiler.compilerPath;
this.cache = compiler.cache;
const options = compiler.options;
this.options = options;
@ -426,7 +430,6 @@ class Compilation {
arrayToSetDeprecation(this.modules, "Compilation.modules");
/** @private @type {Map<string, Module>} */
this._modules = new Map();
this.cache = null;
this.records = null;
/** @type {string[]} */
this.additionalChunkAssets = [];
@ -493,22 +496,21 @@ class Compilation {
if (alreadyAddedModule) {
return callback(null, alreadyAddedModule);
}
const cacheName = "m" + identifier;
if (this.cache && this.cache[cacheName]) {
const cacheModule = this.cache[cacheName];
cacheModule.updateCacheModule(module);
const cacheName = this.compilerPath + "/" + identifier;
this.cache.getModule(cacheName, (err, cacheModule) => {
if (err) return callback(err);
module = cacheModule;
} else {
if (this.cache) {
this.cache[cacheName] = module;
if (cacheModule) {
cacheModule.updateCacheModule(module);
module = cacheModule;
}
}
this._modules.set(identifier, module);
this.modules.add(module);
ModuleGraph.setModuleGraphForModule(module, this.moduleGraph);
return callback(null, module);
this._modules.set(identifier, module);
this.modules.add(module);
ModuleGraph.setModuleGraphForModule(module, this.moduleGraph);
callback(null, module);
});
}
/**
@ -558,8 +560,7 @@ class Compilation {
module.needBuild(
{
fileTimestamps: this.fileTimestamps,
contextTimestamps: this.contextTimestamps
fileSystemInfo: this.fileSystemInfo
},
(err, needBuild) => {
if (err) return callback(err);
@ -592,8 +593,24 @@ class Compilation {
this.hooks.failedModule.call(module, err);
return callback(err);
}
this.hooks.succeedModule.call(module);
return callback();
if (currentProfile !== undefined) {
currentProfile.markStoringStart();
}
this.cache.storeModule(
this.compilerPath + "/" + module.identifier(),
module,
err => {
if (currentProfile !== undefined) {
currentProfile.markStoringEnd();
}
if (err) {
this.hooks.failedModule.call(module, err);
return callback(err);
}
this.hooks.succeedModule.call(module);
return callback();
}
);
}
);
}
@ -1103,38 +1120,49 @@ class Compilation {
this.hooks.beforeModuleAssets.call();
this.createModuleAssets();
if (this.hooks.shouldGenerateChunkAssets.call() !== false) {
this.hooks.beforeChunkAssets.call();
this.createChunkAssets();
}
this.hooks.additionalChunkAssets.call(this.chunks);
this.summarizeDependencies();
if (shouldRecord) {
this.hooks.record.call(this, this.records);
}
this.hooks.additionalAssets.callAsync(err => {
if (err) {
return callback(err);
const cont = () => {
this.hooks.additionalChunkAssets.call(this.chunks);
this.summarizeDependencies();
if (shouldRecord) {
this.hooks.record.call(this, this.records);
}
this.hooks.optimizeChunkAssets.callAsync(this.chunks, err => {
this.hooks.additionalAssets.callAsync(err => {
if (err) {
return callback(err);
}
this.hooks.afterOptimizeChunkAssets.call(this.chunks);
this.hooks.optimizeAssets.callAsync(this.assets, err => {
this.hooks.optimizeChunkAssets.callAsync(this.chunks, err => {
if (err) {
return callback(err);
}
this.hooks.afterOptimizeAssets.call(this.assets);
if (this.hooks.needAdditionalSeal.call()) {
this.unseal();
return this.seal(callback);
}
return this.hooks.afterSeal.callAsync(callback);
this.hooks.afterOptimizeChunkAssets.call(this.chunks);
this.hooks.optimizeAssets.callAsync(this.assets, err => {
if (err) {
return callback(err);
}
this.hooks.afterOptimizeAssets.call(this.assets);
if (this.hooks.needAdditionalSeal.call()) {
this.unseal();
return this.seal(callback);
}
return this.hooks.afterSeal.callAsync(callback);
});
});
});
});
};
if (this.hooks.shouldGenerateChunkAssets.call() !== false) {
this.hooks.beforeChunkAssets.call();
this.createChunkAssets(err => {
if (err) {
return callback(err);
}
cont();
});
} else {
cont();
}
});
}
@ -1996,103 +2024,124 @@ class Compilation {
}
}
createChunkAssets() {
createChunkAssets(callback) {
const outputOptions = this.outputOptions;
const cachedSourceMap = new Map();
const cachedSourceMap = new WeakMap();
/** @type {Map<string, {hash: string, source: Source, chunk: Chunk}>} */
const alreadyWrittenFiles = new Map();
for (const chunk of this.chunks) {
chunk.files = [];
let source;
let file;
let filenameTemplate;
try {
const template = chunk.hasRuntime()
? this.mainTemplate
: this.chunkTemplate;
const manifest = template.getRenderManifest({
chunk,
hash: this.hash,
fullHash: this.fullHash,
outputOptions,
moduleTemplates: this.moduleTemplates,
dependencyTemplates: this.dependencyTemplates,
chunkGraph: this.chunkGraph,
moduleGraph: this.moduleGraph,
runtimeTemplate: this.runtimeTemplate
}); // [{ render(), filenameTemplate, pathOptions, identifier, hash }]
for (const fileManifest of manifest) {
const cacheName = fileManifest.identifier;
const usedHash = fileManifest.hash;
filenameTemplate = fileManifest.filenameTemplate;
file = this.getPath(filenameTemplate, fileManifest.pathOptions);
// check if the same filename was already written by another chunk
const alreadyWritten = alreadyWrittenFiles.get(file);
if (alreadyWritten !== undefined) {
if (alreadyWritten.hash === usedHash) {
if (this.cache) {
this.cache[cacheName] = {
hash: usedHash,
source: alreadyWritten.source
};
}
chunk.files.push(file);
this.hooks.chunkAsset.call(chunk, file);
continue;
} else {
throw new Error(
`Conflict: Multiple chunks emit assets to the same filename ${file}` +
` (chunks ${alreadyWritten.chunk.id} and ${chunk.id})`
);
}
}
if (
this.cache &&
this.cache[cacheName] &&
this.cache[cacheName].hash === usedHash
) {
source = this.cache[cacheName].source;
} else {
source = fileManifest.render();
// Ensure that source is a cached source to avoid additional cost because of repeated access
if (!(source instanceof CachedSource)) {
const cacheEntry = cachedSourceMap.get(source);
if (cacheEntry) {
source = cacheEntry;
} else {
const cachedSource = new CachedSource(source);
cachedSourceMap.set(source, cachedSource);
source = cachedSource;
}
}
if (this.cache) {
this.cache[cacheName] = {
hash: usedHash,
source
};
}
}
if (this.assets[file] && this.assets[file] !== source) {
throw new Error(
`Conflict: Multiple assets emit to the same filename ${file}`
);
}
this.assets[file] = source;
chunk.files.push(file);
this.hooks.chunkAsset.call(chunk, file);
alreadyWrittenFiles.set(file, {
hash: usedHash,
source,
chunk
});
asyncLib.forEach(
this.chunks,
(chunk, _callback) => {
// TODO Workaround for https://github.com/suguru03/neo-async/issues/63
const callback = err => process.nextTick(() => _callback(err));
/** @type {RenderManifestEntry[]} */
let manifest;
try {
chunk.files = [];
const template = chunk.hasRuntime()
? this.mainTemplate
: this.chunkTemplate;
manifest = template.getRenderManifest({
chunk,
hash: this.hash,
fullHash: this.fullHash,
outputOptions,
moduleTemplates: this.moduleTemplates,
dependencyTemplates: this.dependencyTemplates,
chunkGraph: this.chunkGraph,
moduleGraph: this.moduleGraph,
runtimeTemplate: this.runtimeTemplate
}); // [{ render(), filenameTemplate, pathOptions, identifier, hash }]
} catch (err) {
this.errors.push(new ChunkRenderError(chunk, "", err));
return callback();
}
} catch (err) {
this.errors.push(
new ChunkRenderError(chunk, file || filenameTemplate, err)
asyncLib.forEach(
manifest,
(fileManifest, _callback) => {
// TODO Workaround for https://github.com/suguru03/neo-async/issues/63
const callback = err => process.nextTick(() => _callback(err));
const cacheName = this.compilerPath + "/" + fileManifest.identifier;
const usedHash = fileManifest.hash;
this.cache.getAsset(cacheName, usedHash, (err, sourceFromCache) => {
let filenameTemplate, file;
try {
filenameTemplate = fileManifest.filenameTemplate;
file = this.getPath(filenameTemplate, fileManifest.pathOptions);
if (err) {
this.errors.push(
new ChunkRenderError(chunk, file || filenameTemplate, err)
);
return callback();
}
let source = sourceFromCache;
// check if the same filename was already written by another chunk
const alreadyWritten = alreadyWrittenFiles.get(file);
if (alreadyWritten !== undefined) {
if (alreadyWritten.hash !== usedHash) {
return callback(
new Error(
`Conflict: Multiple chunks emit assets to the same filename ${file}` +
` (chunks ${alreadyWritten.chunk.id} and ${chunk.id})`
)
);
} else {
source = alreadyWritten.source;
}
} else if (!source) {
// render the asset
source = fileManifest.render();
// Ensure that source is a cached source to avoid additional cost because of repeated access
if (!(source instanceof CachedSource)) {
const cacheEntry = cachedSourceMap.get(source);
if (cacheEntry) {
source = cacheEntry;
} else {
const cachedSource = new CachedSource(source);
cachedSourceMap.set(source, cachedSource);
source = cachedSource;
}
}
}
if (this.assets[file] && this.assets[file] !== source) {
return callback(
new Error(
`Conflict: Rendering chunk ${chunk.id} ` +
`emits to the filename ${file} ` +
"which was already written to by something else " +
"(but not another chunk)"
)
);
}
this.assets[file] = source;
chunk.files.push(file);
this.hooks.chunkAsset.call(chunk, file);
alreadyWrittenFiles.set(file, {
hash: usedHash,
source,
chunk
});
this.cache.storeAsset(cacheName, usedHash, source, callback);
} catch (err) {
this.errors.push(
new ChunkRenderError(chunk, file || filenameTemplate, err)
);
return callback();
}
});
},
callback
);
}
}
},
callback
);
}
/**

35
lib/Compiler.js
View File

@ -16,15 +16,15 @@ const {
AsyncSeriesHook
} = require("tapable");
const Cache = require("./Cache");
const Compilation = require("./Compilation");
const ConcurrentCompilationError = require("./ConcurrentCompilationError");
const ContextModuleFactory = require("./ContextModuleFactory");
const NormalModuleFactory = require("./NormalModuleFactory");
const RequestShortener = require("./RequestShortener");
const ResolverFactory = require("./ResolverFactory");
const Stats = require("./Stats");
const Watching = require("./Watching");
const ConcurrentCompilationError = require("./ConcurrentCompilationError");
const RequestShortener = require("./RequestShortener");
const { makePathsRelative } = require("./util/identifier");
/** @typedef {import("../declarations/WebpackOptions").Entry} Entry */
@ -100,11 +100,14 @@ class Compiler {
this.name = undefined;
/** @type {Compilation=} */
this.parentCompilation = undefined;
/** @type {Compiler} */
this.root = this;
/** @type {string} */
this.outputPath = "";
this.outputFileSystem = null;
this.inputFileSystem = null;
this.watchFileSystem = null;
/** @type {string|null} */
this.recordsInputPath = null;
@ -125,6 +128,8 @@ class Compiler {
this.requestShortener = new RequestShortener(context);
this.cache = new Cache();
this.compilerPath = "";
/** @type {boolean} */
@ -144,6 +149,7 @@ class Compiler {
if (this.running) return callback(new ConcurrentCompilationError());
const finalCallback = (err, stats) => {
this.cache.beginIdle();
this.running = false;
if (callback !== undefined) return callback(err, stats);
@ -201,16 +207,20 @@ class Compiler {
});
};
this.hooks.beforeRun.callAsync(this, err => {
this.cache.endIdle(err => {
if (err) return finalCallback(err);
this.hooks.run.callAsync(this, err => {
this.hooks.beforeRun.callAsync(this, err => {
if (err) return finalCallback(err);
this.readRecords(err => {
this.hooks.run.callAsync(this, err => {
if (err) return finalCallback(err);
this.compile(onCompiled);
this.readRecords(err => {
if (err) return finalCallback(err);
this.compile(onCompiled);
});
});
});
});
@ -400,7 +410,11 @@ class Compiler {
childCompiler.compilerPath =
this.compilerPath + "/" + compilerName + compilerIndex;
const relativeCompilerName = makePathsRelative(this.context, compilerName);
const relativeCompilerName = makePathsRelative(
this.context,
compilerName,
this.root
);
if (!this.records[relativeCompilerName]) {
this.records[relativeCompilerName] = [];
}
@ -416,6 +430,7 @@ class Compiler {
childCompiler.options.output[name] = outputOptions[name];
}
childCompiler.parentCompilation = compilation;
childCompiler.root = this.root;
compilation.hooks.childCompiler.call(
childCompiler,
@ -497,6 +512,10 @@ class Compiler {
});
});
}
close(callback) {
this.cache.shutdown(callback);
}
}
module.exports = Compiler;

13
lib/ContextModule.js
View File

@ -231,14 +231,13 @@ class ContextModule extends Module {
* @param {function(WebpackError=, boolean=): void} callback callback function, returns true, if the module needs a rebuild
* @returns {void}
*/
needBuild({ contextTimestamps }, callback) {
if (this._forceBuild || !contextTimestamps) return callback(null, true);
const ts = contextTimestamps.get(this.context);
if (!ts) {
return callback(null, true);
}
needBuild({ fileSystemInfo }, callback) {
if (this._forceBuild) return callback(null, true);
fileSystemInfo.getContextTimestamp(this.context, (err, info) => {
if (err || !info) return callback(null, true);
return callback(null, ts >= this.buildInfo.builtTime);
return callback(null, info.safeTime >= this.buildInfo.builtTime);
});
}
/**

3
lib/DllReferencePlugin.js
View File

@ -64,7 +64,8 @@ class DllReferencePlugin {
// be added as a compilation error later on.
const manifestPath = makePathsRelative(
compiler.options.context,
manifest
manifest,
compiler.root
);
params[
"dll reference parse error " + manifest

134
lib/FileSystemInfo.js Normal file
View File

@ -0,0 +1,134 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
const AsyncQueue = require("./util/AsyncQueue");
let FS_ACCURACY = 2000;
/**
* @typedef {Object} FileSystemInfoEntry
* @property {number} safeTime
* @property {number} timestamp
*/
/* istanbul ignore next */
const applyMtime = mtime => {
if (FS_ACCURACY > 1 && mtime % 2 !== 0) FS_ACCURACY = 1;
else if (FS_ACCURACY > 10 && mtime % 20 !== 0) FS_ACCURACY = 10;
else if (FS_ACCURACY > 100 && mtime % 200 !== 0) FS_ACCURACY = 100;
else if (FS_ACCURACY > 1000 && mtime % 2000 !== 0) FS_ACCURACY = 1000;
};
class FileSystemInfo {
constructor(fs) {
this.fs = fs;
this._fileTimestamps = new Map();
this._contextTimestamps = new Map();
this.fileTimestampQueue = new AsyncQueue({
name: "file timestamp",
parallelism: 30,
processor: this._readFileTimestamp.bind(this)
});
this.contextTimestampQueue = new AsyncQueue({
name: "context timestamp",
parallelism: 2,
processor: this._readContextTimestamp.bind(this)
});
}
/**
* @param {Map<string, FileSystemInfoEntry>} map timestamps
* @returns {void}
*/
addFileTimestamps(map) {
for (const [path, ts] of map) {
this._fileTimestamps.set(path, ts);
}
}
/**
* @param {Map<string, FileSystemInfoEntry>} map timestamps
* @returns {void}
*/
addContextTimestamps(map) {
for (const [path, ts] of map) {
this._contextTimestamps.set(path, ts);
}
}
/**
* @param {string} path file path
* @param {function(Error=, FileSystemInfoEntry=): void} callback callback function
* @returns {void}
*/
getFileTimestamp(path, callback) {
const cache = this._fileTimestamps.get(path);
if (cache !== undefined) return cache;
this.fileTimestampQueue.add(path, callback);
}
/**
* @param {string} path context path
* @param {function(Error=, FileSystemInfoEntry=): void} callback callback function
* @returns {void}
*/
getContextTimestamp(path, callback) {
const cache = this._contextTimestamps.get(path);
if (cache !== undefined) return cache;
this.contextTimestampQueue.add(path, callback);
}
// TODO getFileHash(path, callback)
_readFileTimestamp(path, callback) {
this.fs.stat(path, (err, stat) => {
if (err) {
if (err.code === "ENOENT") {
this._fileTimestamps.set(path, null);
return callback(null, null);
}
return callback(err);
}
if (stat.mtime) applyMtime(+stat.mtime);
const mtime = +stat.mtime || Infinity;
const ts = {
safeTime: mtime + FS_ACCURACY,
timestamp: mtime
};
this._fileTimestamps.set(path, ts);
callback(null, ts);
});
}
_readContextTimestamp(path, callback) {
// TODO read whole folder
this._contextTimestamps.set(path, null);
callback(null, null);
}
getDeprecatedFileTimestamps() {
const map = new Map();
for (const [path, info] of this._fileTimestamps) {
if (info) map.set(path, info.safeTime);
}
return map;
}
getDeprecatedContextTimestamps() {
const map = new Map();
for (const [path, info] of this._contextTimestamps) {
if (info) map.set(path, info.safeTime);
}
return map;
}
}
module.exports = FileSystemInfo;

12
lib/Module.js
View File

@ -17,6 +17,7 @@ const { compareChunksById } = require("./util/comparators");
/** @typedef {import("./Compilation")} Compilation */
/** @typedef {import("./Dependency")} Dependency */
/** @typedef {import("./DependencyTemplates")} DependencyTemplates */
/** @typedef {import("./FileSystemInfo")} FileSystemInfo */
/** @typedef {import("./RequestShortener")} RequestShortener */
/** @typedef {import("./RuntimeTemplate")} RuntimeTemplate */
/** @typedef {import("./WebpackError")} WebpackError */
@ -45,8 +46,7 @@ const { compareChunksById } = require("./util/comparators");
/**
* @typedef {Object} NeedBuildContext
* @property {Map<string, number>=} fileTimestamps
* @property {Map<string, number>=} contextTimestamps
* @property {FileSystemInfo} fileSystemInfo
*/
/** @typedef {KnownBuildMeta & Record<string, any>} BuildMeta */
@ -460,9 +460,11 @@ class Module extends DependenciesBlock {
callback(
null,
!this.buildMeta ||
!context.fileTimestamps ||
!context.contextTimestamps ||
this.needRebuild(context.fileTimestamps, context.contextTimestamps)
this.needRebuild === Module.prototype.needRebuild ||
this.needRebuild(
context.fileSystemInfo.getDeprecatedFileTimestamps(),
context.fileSystemInfo.getDeprecatedContextTimestamps()
)
);
}

9
lib/ModuleProfile.js
View File

@ -42,6 +42,15 @@ class ModuleProfile {
this.building = this.buildingEndTime - this.buildingStartTime;
}
markStoringStart() {
this.storingStartTime = Date.now();
}
markStoringEnd() {
this.storingEndTime = Date.now();
this.storing = this.storingEndTime - this.storingStartTime;
}
mergeInto(realProfile) {
if (this.factory > realProfile.additionalFactories)
realProfile.additionalFactories = this.factory;

10
lib/MultiCompiler.js
View File

@ -280,4 +280,14 @@ module.exports = class MultiCompiler {
}
}
}
close(callback) {
asyncLib.each(
this.compilers,
(compiler, callback) => {
compiler.close(callback);
},
callback
);
}
};

56
lib/NormalModule.js
View File

@ -6,6 +6,7 @@
"use strict";
const { getContext, runLoaders } = require("loader-runner");
const asyncLib = require("neo-async");
const {
CachedSource,
LineToLineMappedSource,
@ -13,7 +14,6 @@ const {
RawSource,
SourceMapSource
} = require("webpack-sources");
const Module = require("./Module");
const ModuleBuildError = require("./ModuleBuildError");
const ModuleError = require("./ModuleError");
@ -36,6 +36,8 @@ const contextify = require("./util/identifier").contextify;
/** @typedef {import("./WebpackError")} WebpackError */
/** @typedef {import("./util/createHash").Hash} Hash */
const EARLY_RETURN_ERROR = new Error("flags early return is not an error");
const asString = buf => {
if (Buffer.isBuffer(buf)) {
return buf.toString("utf-8");
@ -569,7 +571,7 @@ class NormalModule extends Module {
* @param {function(WebpackError=, boolean=): void} callback callback function, returns true, if the module needs a rebuild
* @returns {void}
*/
needBuild({ fileTimestamps, contextTimestamps }, callback) {
needBuild({ fileSystemInfo }, callback) {
// build if enforced
if (this._forceBuild) return callback(null, true);
@ -579,27 +581,41 @@ class NormalModule extends Module {
// always build when module is not cacheable
if (!this.buildInfo.cacheable) return callback(null, true);
// missing information
if (!fileTimestamps && this.buildInfo.fileDependencies.size > 0)
return callback(null, true);
if (!contextTimestamps && this.buildInfo.contextDependencies.size > 0)
return callback(null, true);
// Check timestamps of all dependencies
// Missing timestamp -> need build
// Timestamp bigger than buildTimestamp -> need build
for (const file of this.buildInfo.fileDependencies) {
const timestamp = fileTimestamps.get(file);
if (!timestamp) return callback(null, true);
if (timestamp >= this.buildTimestamp) return callback(null, true);
}
for (const file of this.buildInfo.contextDependencies) {
const timestamp = contextTimestamps.get(file);
if (!timestamp) return callback(null, true);
if (timestamp >= this.buildTimestamp) return callback(null, true);
}
// elsewise -> no build needed
return callback(null, false);
asyncLib.parallel(
[
callback =>
asyncLib.each(
this.buildInfo.fileDependencies,
(file, callback) => {
fileSystemInfo.getFileTimestamp(file, (err, info) => {
if (err) return callback(err);
if (!info || info.safeTime >= this.buildTimestamp)
return callback(EARLY_RETURN_ERROR);
callback();
});
},
callback
),
callback =>
asyncLib.each(
this.buildInfo.contextDependencies,
(context, callback) => {
fileSystemInfo.getContextTimestamp(context, (err, info) => {
if (err) return callback(err);
if (!info || info.safeTime >= this.buildTimestamp)
return callback(EARLY_RETURN_ERROR);
callback();
});
},
callback
)
],
err =>
err === EARLY_RETURN_ERROR ? callback(null, true) : callback(err, false)
);
}
/**

6
lib/RecordIdsPlugin.js
View File

@ -68,7 +68,7 @@ class RecordIdsPlugin {
? identifierUtils.makePathsRelative(
compiler.context,
module.identifier(),
compilation.cache
compiler.root
)
: module.identifier();
records.modules.byIdentifier[identifier] = moduleId;
@ -97,7 +97,7 @@ class RecordIdsPlugin {
? identifierUtils.makePathsRelative(
compiler.context,
module.identifier(),
compilation.cache
compiler.root
)
: module.identifier();
const id = records.modules.byIdentifier[identifier];
@ -122,7 +122,7 @@ class RecordIdsPlugin {
return identifierUtils.makePathsRelative(
compiler.context,
module.identifier(),
compilation.cache
compiler.root
);
}
return module.identifier();

2
lib/Stats.js
View File

@ -834,7 +834,7 @@ class Stats {
obj.name = identifierUtils.makePathsRelative(
context,
child.name,
compilation.cache
compilation.compiler.root
);
}
return obj;

80
lib/Watching.js
View File

@ -7,7 +7,14 @@
const Stats = require("./Stats");
/** @typedef {import("./Compiler")} Compiler */
class Watching {
/**
* @param {Compiler} compiler the compiler
* @param {TODO} watchOptions options
* @param {TODO} handler TODO
*/
constructor(compiler, watchOptions, handler) {
this.startTime = null;
this.invalid = false;
@ -38,44 +45,47 @@ class Watching {
this.startTime = Date.now();
this.running = true;
this.invalid = false;
this.compiler.hooks.watchRun.callAsync(this.compiler, err => {
this.compiler.cache.endIdle(err => {
if (err) return this._done(err);
const onCompiled = (err, compilation) => {
this.compiler.hooks.watchRun.callAsync(this.compiler, err => {
if (err) return this._done(err);
if (this.invalid) return this._done();
if (this.compiler.hooks.shouldEmit.call(compilation) === false) {
return this._done(null, compilation);
}
this.compiler.emitAssets(compilation, err => {
const onCompiled = (err, compilation) => {
if (err) return this._done(err);
if (this.invalid) return this._done();
this.compiler.emitRecords(err => {
if (err) return this._done(err);
if (compilation.hooks.needAdditionalPass.call()) {
compilation.needAdditionalPass = true;
const stats = new Stats(compilation);
stats.startTime = this.startTime;
stats.endTime = Date.now();
this.compiler.hooks.done.callAsync(stats, err => {
if (err) return this._done(err);
this.compiler.hooks.additionalPass.callAsync(err => {
if (err) return this._done(err);
this.compiler.compile(onCompiled);
});
});
return;
}
if (this.compiler.hooks.shouldEmit.call(compilation) === false) {
return this._done(null, compilation);
}
this.compiler.emitAssets(compilation, err => {
if (err) return this._done(err);
if (this.invalid) return this._done();
this.compiler.emitRecords(err => {
if (err) return this._done(err);
if (compilation.hooks.needAdditionalPass.call()) {
compilation.needAdditionalPass = true;
const stats = new Stats(compilation);
stats.startTime = this.startTime;
stats.endTime = Date.now();
this.compiler.hooks.done.callAsync(stats, err => {
if (err) return this._done(err);
this.compiler.hooks.additionalPass.callAsync(err => {
if (err) return this._done(err);
this.compiler.compile(onCompiled);
});
});
return;
}
return this._done(null, compilation);
});
});
});
};
this.compiler.compile(onCompiled);
};
this.compiler.compile(onCompiled);
});
});
}
@ -93,11 +103,13 @@ class Watching {
const stats = compilation ? this._getStats(compilation) : null;
if (err) {
this.compiler.hooks.failed.call(err);
this.compiler.cache.beginIdle();
this.handler(err, stats);
return;
}
this.compiler.hooks.done.callAsync(stats, () => {
this.compiler.cache.beginIdle();
this.handler(null, stats);
if (!this.closed) {
this.watch(
@ -169,9 +181,11 @@ class Watching {
close(callback) {
const finalCallback = () => {
this.compiler.hooks.watchClose.call();
this.compiler.running = false;
if (callback !== undefined) callback();
this.compiler.cache.shutdown(err => {
this.compiler.hooks.watchClose.call();
if (callback !== undefined) callback(err);
});
};
this.closed = true;

6
lib/WebpackOptionsApply.js
View File

@ -457,10 +457,8 @@ class WebpackOptionsApply extends OptionsApply {
new WarnCaseSensitiveModulesPlugin().apply(compiler);
if (options.cache) {
const CachePlugin = require("./CachePlugin");
new CachePlugin(
typeof options.cache === "object" ? options.cache : null
).apply(compiler);
const MemoryCachePlugin = require("./cache/MemoryCachePlugin");
new MemoryCachePlugin().apply(compiler);
}
compiler.hooks.afterPlugins.call(compiler);

48
lib/cache/MemoryCachePlugin.js vendored Normal file
View File

@ -0,0 +1,48 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
/** @typedef {import("webpack-sources").Source} Source */
/** @typedef {import("../Compiler")} Compiler */
/** @typedef {import("../Module")} Module */
class MemoryCachePlugin {
/**
* @param {Compiler} compiler Webpack compiler
* @returns {void}
*/
apply(compiler) {
/** @type {Map<string, Module>} */
const moduleCache = new Map();
/** @type {Map<string, { hash: string, source: Source }>} */
const assetCache = new Map();
compiler.cache.hooks.storeModule.tap(
"MemoryCachePlugin",
(identifier, module) => {
moduleCache.set(identifier, module);
}
);
compiler.cache.hooks.getModule.tap("MemoryCachePlugin", identifier => {
return moduleCache.get(identifier);
});
compiler.cache.hooks.storeAsset.tap(
"MemoryCachePlugin",
(identifier, hash, source) => {
assetCache.set(identifier, { hash, source });
}
);
compiler.cache.hooks.getAsset.tap(
"MemoryCachePlugin",
(identifier, hash) => {
const cacheEntry = assetCache.get(identifier);
if (cacheEntry !== undefined && cacheEntry.hash === hash) {
return cacheEntry.source;
}
}
);
}
}
module.exports = MemoryCachePlugin;

2
lib/optimize/AggressiveSplittingPlugin.js
View File

@ -106,7 +106,7 @@ class AggressiveSplittingPlugin {
const name = identifierUtils.makePathsRelative(
compiler.context,
m.identifier(),
compilation.cache
compiler.root
);
nameToModuleMap.set(name, m);
moduleToNameMap.set(m, name);

15
lib/util/identifier.js
View File

@ -50,18 +50,23 @@ const _makePathsRelative = (context, identifier) => {
.join("");
};
const makePathsRelativeCache = new WeakMap();
/**
*
* @param {string} context context used to create relative path
* @param {string} identifier identifier used to create relative path
* @param {MakeRelativePathsCache=} cache the cache object being set
* @param {Object=} associatedObjectForCache an object to which the cache will be attached
* @returns {string} the returned relative path
*/
exports.makePathsRelative = (context, identifier, cache) => {
if (!cache) return _makePathsRelative(context, identifier);
exports.makePathsRelative = (context, identifier, associatedObjectForCache) => {
if (!associatedObjectForCache) return _makePathsRelative(context, identifier);
const relativePaths =
cache.relativePaths || (cache.relativePaths = new Map());
let relativePaths = makePathsRelativeCache.get(associatedObjectForCache);
if (relativePaths === undefined) {
relativePaths = new Map();
makePathsRelativeCache.set(associatedObjectForCache, relativePaths);
}
let cachedResult;
let contextCache = relativePaths.get(context);

11
lib/webpack.js
View File

@ -17,6 +17,7 @@ const NodeEnvironmentPlugin = require("./node/NodeEnvironmentPlugin");
const validateSchema = require("./validateSchema");
/** @typedef {import("../declarations/WebpackOptions").WebpackOptions} WebpackOptions */
/** @typedef {import("./Stats")} Stats */
/**
* @param {WebpackOptions} options options object
@ -68,7 +69,11 @@ const webpack = (options, callback) => {
: options.watchOptions || {};
return compiler.watch(watchOptions, callback);
}
compiler.run(callback);
compiler.run((err, stats) => {
compiler.close(err2 => {
callback(err || err2, stats);
});
});
}
return compiler;
};
@ -99,7 +104,6 @@ const exportPlugins = (obj, mappings) => {
exportPlugins(exports, {
AutomaticPrefetchPlugin: () => require("./AutomaticPrefetchPlugin"),
BannerPlugin: () => require("./BannerPlugin"),
CachePlugin: () => require("./CachePlugin"),
ContextExclusionPlugin: () => require("./ContextExclusionPlugin"),
ContextReplacementPlugin: () => require("./ContextReplacementPlugin"),
DefinePlugin: () => require("./DefinePlugin"),
@ -137,6 +141,9 @@ exportPlugins(exports, {
UmdMainTemplatePlugin: () => require("./UmdMainTemplatePlugin"),
WatchIgnorePlugin: () => require("./WatchIgnorePlugin")
});
exportPlugins((exports.cache = {}), {
MemoryCachePlugin: () => require("./cache/MemoryCachePlugin")
});
exportPlugins((exports.dependencies = {}), {
DependencyReference: () => require("./dependencies/DependencyReference")
});

40
test/CachePlugin.unittest.js
View File

@ -1,40 +0,0 @@
"use strict";
const CachePlugin = require("../lib/CachePlugin");
describe("CachePlugin", () => {
let env;
beforeEach(() => {
env = {
compilation: {
compiler: {},
warnings: []
}
};
});
describe("applyMtime", () => {
beforeEach(() => (env.plugin = new CachePlugin()));
it("sets file system accuracy to 1 for granular modification timestamp", () => {
env.plugin.applyMtime(1483819067001);
expect(env.plugin.FS_ACCURACY).toBe(1);
});
it("sets file system accuracy to 10 for moderately granular modification timestamp", () => {
env.plugin.applyMtime(1483819067004);
expect(env.plugin.FS_ACCURACY).toBe(10);
});
it("sets file system accuracy to 100 for moderately coarse modification timestamp", () => {
env.plugin.applyMtime(1483819067040);
expect(env.plugin.FS_ACCURACY).toBe(100);
});
it("sets file system accuracy to 1000 for coarse modification timestamp", () => {
env.plugin.applyMtime(1483819067400);
expect(env.plugin.FS_ACCURACY).toBe(1000);
});
});
});

97
test/NormalModule.unittest.js
View File

@ -190,103 +190,6 @@ describe("NormalModule", () => {
expect(normalModule.hasDependencies()).toBe(false);
});
});
describe("#needBuild", () => {
let fileTimestamps;
let contextTimestamps;
let fileDependencies;
let contextDependencies;
let fileA;
let fileB;
function setDeps(fileDependencies, contextDependencies) {
normalModule.buildInfo.fileDependencies = fileDependencies;
normalModule.buildInfo.contextDependencies = contextDependencies;
}
beforeEach(() => {
fileA = "fileA";
fileB = "fileB";
fileDependencies = [fileA, fileB];
contextDependencies = [fileA, fileB];
fileTimestamps = new Map([[fileA, 1], [fileB, 1]]);
contextTimestamps = new Map([[fileA, 1], [fileB, 1]]);
normalModule.buildTimestamp = 2;
normalModule._forceBuild = false;
setDeps(fileDependencies, contextDependencies);
});
describe("given all timestamps are older than the buildTimestamp", () => {
it("returns false", done => {
normalModule.needBuild(
{ fileTimestamps, contextTimestamps },
(err, result) => {
if (err) return done(err);
expect(result).toBe(false);
done();
}
);
});
});
describe("given a file timestamp is newer than the buildTimestamp", () => {
beforeEach(() => {
fileTimestamps.set(fileA, 3);
});
it("returns true", done => {
normalModule.needBuild(
{ fileTimestamps, contextTimestamps },
(err, result) => {
if (err) return done(err);
expect(result).toBe(true);
done();
}
);
});
});
describe("given a no file timestamp exists", () => {
beforeEach(() => {
fileTimestamps = new Map();
});
it("returns true", done => {
normalModule.needBuild(
{ fileTimestamps, contextTimestamps },
(err, result) => {
if (err) return done(err);
expect(result).toBe(true);
done();
}
);
});
});
describe("given a context timestamp is newer than the buildTimestamp", () => {
beforeEach(() => {
contextTimestamps.set(fileA, 3);
});
it("returns true", done => {
normalModule.needBuild(
{ fileTimestamps, contextTimestamps },
(err, result) => {
if (err) return done(err);
expect(result).toBe(true);
done();
}
);
});
});
describe("given a no context timestamp exists", () => {
beforeEach(() => {
contextTimestamps = new Map();
});
it("returns true", done => {
normalModule.needBuild(
{ fileTimestamps, contextTimestamps },
(err, result) => {
if (err) return done(err);
expect(result).toBe(true);
done();
}
);
});
});
});
describe("#applyNoParseRule", () => {
let rule;

6
test/watchCases/cache/child-compilation-cache/0/index.js vendored
View File

@ -2,13 +2,13 @@ it("should use correct caches in compilation and child compilations", function()
var x = require("./report-cache-counters-loader!./changing-file");
switch(WATCH_STEP) {
case "0":
expect(x).toEqual([1, 1]);
expect(x).toEqual(["", 1, "/my-compiler 1230", 1]);
break;
case "1":
expect(x).toEqual([2, 1]);
expect(x).toEqual(["", 2, "/my-compiler 4560", 1]);
break;
case "2":
expect(x).toEqual([3, 2]);
expect(x).toEqual(["", 3, "/my-compiler 1230", 2]);
break;
default:
throw new Error("Not handled step");

30
test/watchCases/cache/child-compilation-cache/0/report-cache-counters-loader.js vendored
View File

@ -1,9 +1,26 @@
var map = new Map();
var currentWatchStepModule = require("../../../../helpers/currentWatchStep");
var cacheMap = new WeakMap();
const getCache = (associate, path) => {
let o = cacheMap.get(associate);
if(o === undefined) {
o = new Map();
cacheMap.set(associate, o);
}
let c = o.get(path);
if(c === undefined) {
c = { counter: 0 };
o.set(path, c);
}
return c;
};
module.exports = function(source) {
if(map.has(currentWatchStepModule.step)) return map.get(currentWatchStepModule.step);
this._compilation.cache.counter = (this._compilation.cache.counter || 0) + 1;
const compilationCache = getCache(this._compiler.root, this._compilation.compilerPath);
compilationCache.counter++;
var childCompiler = this._compilation.createChildCompiler("my-compiler " + source.trim(), {
filename: "test"
@ -12,10 +29,15 @@ module.exports = function(source) {
childCompiler.runAsChild((err, entries, compilation) => {
if(err) return callback(err);
var childCache = compilation.cache;
childCache.counter = (childCache.counter || 0) + 1;
const childCache = getCache(this._compiler.root, compilation.compilerPath);
childCache.counter++;
var result = `module.exports = [${this._compilation.cache.counter}, ${childCache.counter}]; // ${source}`;
var result = `module.exports = ${JSON.stringify([
this._compilation.compilerPath,
compilationCache.counter,
compilation.compilerPath,
childCache.counter
])}; // ${source}`;
map.set(currentWatchStepModule.step, result);
callback(null, result);
});