store snapshot of resolving dependencies and re-resolve when needed

to be able to invalidate cache when PnP build dependencies change

cacheDirectory defaults to .pnp/.cache/webpack when PnP is used
fallback cacheDirectory is now .cache/webpack instead of tmpdir
This commit is contained in:
Tobias Koppers 2019-08-19 21:57:49 +02:00
parent a2caa36b8e
commit cb4c02ebe1
5 changed files with 319 additions and 111 deletions

View File

@ -27,6 +27,8 @@ const RBDT_FILE = 3;
const RBDT_DIRECTORY_DEPENDENCIES = 4;
const RBDT_FILE_DEPENDENCIES = 5;
const INVALID = Symbol("invalid");
/**
* @typedef {Object} FileSystemInfoEntry
* @property {number} safeTime
@ -223,47 +225,80 @@ class FileSystemInfo {
const files = new Set();
const directories = new Set();
const missing = new Set();
const resolveFiles = new Set();
const resolveDirectories = new Set();
const resolveMissing = new Set();
const resolveResults = new Map();
/** @type {asyncLib.QueueObject<{type: number, path: string, context?: string }, Error>} */
const queue = asyncLib.queue(({ type, context, path }, callback) => {
switch (type) {
case RBDT_RESOLVE: {
const isDirectory = /[\\/]$/.test(path);
const isDeps = /^deps:/.test(path);
if (isDeps) path = path.slice(5);
if (isDirectory) {
resolveContext(
context,
path.replace(/[\\/]$/, ""),
(err, result) => {
if (err) return callback(err);
queue.push({
type: isDeps ? RBDT_DIRECTORY_DEPENDENCIES : RBDT_DIRECTORY,
path: result
});
callback();
}
);
} else {
resolve(context, path, (err, result) => {
if (err) return callback(err);
queue.push({
type: isDeps ? RBDT_FILE_DEPENDENCIES : RBDT_FILE,
path: result
});
callback();
});
}
break;
const resolveDirectory = path => {
const key = `d\n${context}\n${path}`;
if (resolveResults.has(key)) {
return callback();
}
case RBDT_RESOLVE_DIRECTORY: {
resolveContext(context, path, (err, result) => {
if (err) return callback(err);
resolveContext(
context,
path,
{
fileDependencies: resolveFiles,
contextDependencies: resolveDirectories,
missingDependencies: resolveMissing
},
(err, result) => {
if (err) {
if (err.code === "ENOENT" || err.code === "UNDECLARED_DEPENDENCY")
return callback();
return callback(err);
}
resolveResults.set(key, result);
queue.push({
type: RBDT_DIRECTORY,
path: result
});
callback();
});
}
);
};
const resolveFile = path => {
const key = `f\n${context}\n${path}`;
if (resolveResults.has(key)) {
return callback();
}
resolve(
context,
path,
{
fileDependencies: resolveFiles,
contextDependencies: resolveDirectories,
missingDependencies: resolveMissing
},
(err, result) => {
if (err) {
if (err.code === "ENOENT" || err.code === "UNDECLARED_DEPENDENCY")
return callback();
return callback(err);
}
resolveResults.set(key, result);
queue.push({
type: RBDT_FILE,
path: result
});
callback();
}
);
};
switch (type) {
case RBDT_RESOLVE: {
const isDirectory = /[\\/]$/.test(path);
if (isDirectory) {
resolveDirectory(path.slice(0, path.length - 1));
} else {
resolveFile(path);
}
break;
}
case RBDT_RESOLVE_DIRECTORY: {
resolveDirectory(path);
break;
}
case RBDT_FILE: {
@ -273,6 +308,9 @@ class FileSystemInfo {
}
this.fs.realpath(path, (err, realPath) => {
if (err) return callback(err);
if (realPath !== path) {
resolveFiles.add(path);
}
if (!files.has(realPath)) {
files.add(realPath);
queue.push({
@ -291,6 +329,9 @@ class FileSystemInfo {
}
this.fs.realpath(path, (err, realPath) => {
if (err) return callback(err);
if (realPath !== path) {
resolveFiles.add(path);
}
if (!directories.has(realPath)) {
directories.add(realPath);
queue.push({
@ -336,6 +377,7 @@ class FileSystemInfo {
this.fs.readFile(packageJson, (err, content) => {
if (err) {
if (err.code === "ENOENT") {
resolveMissing.add(packageJson);
const parent = dirname(this.fs, packagePath);
if (parent !== packagePath) {
queue.push({
@ -348,6 +390,7 @@ class FileSystemInfo {
}
return callback(err);
}
resolveFiles.add(packageJson);
let packageData;
try {
packageData = JSON.parse(content.toString("utf-8"));
@ -371,7 +414,17 @@ class FileSystemInfo {
}
}, 50);
queue.drain = () => {
callback(null, { files, directories, missing });
callback(null, {
files,
directories,
missing,
resolveResults,
resolveDependencies: {
files: resolveFiles,
directories: resolveDirectories,
missing: resolveMissing
}
});
};
queue.error = err => {
callback(err);
@ -386,6 +439,53 @@ class FileSystemInfo {
}
}
/**
* @param {Map<string, string>} resolveResults results from resolving
* @param {function(Error=, boolean=): void} callback callback with true when resolveResults resolve the same way
* @returns {void}
*/
checkResolveResultsValid(resolveResults, callback) {
asyncLib.eachLimit(
resolveResults,
20,
([key, expectedResult], callback) => {
const [type, context, path] = key.split("\n");
switch (type) {
case "d":
resolveContext(context, path, (err, result) => {
if (err) return callback(err);
if (result !== expectedResult) return callback(INVALID);
callback();
});
break;
case "f":
resolve(context, path, (err, result) => {
if (err) return callback(err);
if (result !== expectedResult) return callback(INVALID);
callback();
});
break;
default:
callback(new Error("Unexpected type in resolve result key"));
break;
}
},
/**
* @param {Error | typeof INVALID=} err error or invalid flag
* @returns {void}
*/
err => {
if (err === INVALID) {
return callback(null, false);
}
if (err) {
return callback(err);
}
return callback(null, true);
}
);
}
/**
*
* @param {number} startTime when processing the files has started

View File

@ -5,9 +5,8 @@
"use strict";
const findCacheDir = require("find-cache-dir");
const os = require("os");
const path = require("path");
const pkgDir = require("pkg-dir");
const OptionsDefaulter = require("./OptionsDefaulter");
const Template = require("./Template");
@ -71,8 +70,19 @@ class WebpackOptionsDefaulter extends OptionsDefaulter {
value.version = "";
}
if (value.cacheDirectory === undefined) {
value.cacheDirectory =
findCacheDir({ name: "webpack" }) || os.tmpdir();
const cwd = process.cwd();
const dir = pkgDir.sync(cwd);
if (!dir) {
value.cacheDirectory = path.resolve(dir, ".cache/webpack");
// @ts-ignore
} else if (process.versions.pnp) {
value.cacheDirectory = path.resolve(dir, ".pnp/.cache/webpack");
} else {
value.cacheDirectory = path.resolve(
dir,
"node_modules/.cache/webpack"
);
}
}
if (value.cacheLocation === undefined) {
value.cacheLocation = path.resolve(value.cacheDirectory, value.name);

View File

@ -18,21 +18,33 @@ const {
const MAX_INLINE_SIZE = 20000;
class DataWithBuildSnapshot {
constructor(data, buildSnapshot, buildDependencies) {
constructor(
data,
buildSnapshot,
buildDependencies,
resolveResults,
resolveBuildDependenciesSnapshot
) {
this.data = data;
this.buildSnapshot = buildSnapshot;
this.buildDependencies = buildDependencies;
this.resolveResults = resolveResults;
this.resolveBuildDependenciesSnapshot = resolveBuildDependenciesSnapshot;
}
serialize({ write }) {
write(this.buildSnapshot);
write(this.buildDependencies);
write(this.resolveResults);
write(this.resolveBuildDependenciesSnapshot);
write(this.data);
}
deserialize({ read }) {
this.buildSnapshot = read();
this.buildDependencies = read();
this.resolveResults = read();
this.resolveBuildDependenciesSnapshot = read();
this.data = read();
}
}
@ -40,7 +52,7 @@ class DataWithBuildSnapshot {
makeSerializable(
DataWithBuildSnapshot,
"webpack/lib/cache/PackFileCacheStrategy",
"DataWithBuildSnapshot"
"DataWithBuildSnapshot_v1"
);
class Pack {
@ -244,43 +256,107 @@ class PackFileCacheStrategy {
});
this.context = context;
this.cacheLocation = cacheLocation;
this.version = version;
this.logger = logger;
this.buildDependencies = new Set();
this.newBuildDependencies = new LazySet();
this.resolveBuildDependenciesSnapshot = undefined;
this.resolveResults = undefined;
this.buildSnapshot = undefined;
this.oldBuildDependencies = new Set();
this.buildDependencies = new LazySet();
this.packPromise = this._openPack();
}
_openPack() {
const { logger, cacheLocation, version } = this;
let buildSnapshot;
let buildDependencies;
let newBuildDependencies;
let resolveBuildDependenciesSnapshot;
let resolveResults;
logger.time("restore pack");
this.packPromise = this.fileSerializer
return this.fileSerializer
.deserialize({ filename: `${cacheLocation}.pack`, logger })
.catch(err => {
if (err.code !== "ENOENT") {
logger.warn(
`Restoring pack failed from ${cacheLocation}.pack: ${err}`
);
logger.debug(err.stack);
} else {
logger.debug(`No pack exists at ${cacheLocation}.pack: ${err}`);
}
return undefined;
})
.then(cacheEntry => {
logger.timeEnd("restore pack");
if (cacheEntry instanceof DataWithBuildSnapshot) {
logger.timeEnd("restore pack");
return new Promise((resolve, reject) => {
logger.time("check build dependencies");
this.fileSystemInfo.checkSnapshotValid(
cacheEntry.buildSnapshot,
(err, valid) => {
if (err) return reject(err);
logger.timeEnd("check build dependencies");
if (!valid) {
logger.log(
`Restored pack from ${cacheLocation}.pack, but build dependencies have changed.`
);
return resolve(undefined);
logger.time("check build dependencies");
return Promise.all([
new Promise((resolve, reject) => {
this.fileSystemInfo.checkSnapshotValid(
cacheEntry.buildSnapshot,
(err, valid) => {
if (err) return reject(err);
if (!valid) {
logger.log(
`Restored pack from ${cacheLocation}.pack, but build dependencies have changed.`
);
return resolve(false);
}
buildSnapshot = cacheEntry.buildSnapshot;
return resolve(true);
}
buildSnapshot = cacheEntry.buildSnapshot;
buildDependencies = cacheEntry.buildDependencies;
);
}),
new Promise((resolve, reject) => {
this.fileSystemInfo.checkSnapshotValid(
cacheEntry.resolveBuildDependenciesSnapshot,
(err, valid) => {
if (err) return reject(err);
if (valid) {
resolveBuildDependenciesSnapshot =
cacheEntry.resolveBuildDependenciesSnapshot;
buildDependencies = cacheEntry.buildDependencies;
resolveResults = cacheEntry.resolveResults;
return resolve(true);
}
logger.debug(
"resolving of build dependencies is invalid, will re-resolve build dependencies"
);
this.fileSystemInfo.checkResolveResultsValid(
cacheEntry.resolveResults,
(err, valid) => {
if (err) return reject(err);
if (valid) {
newBuildDependencies = cacheEntry.buildDependencies;
resolveResults = cacheEntry.resolveResults;
return resolve(true);
}
logger.log(
`Restored pack from ${cacheLocation}.pack, but build dependencies resolve to different locations.`
);
return resolve(false);
}
);
}
);
})
])
.catch(err => {
logger.timeEnd("check build dependencies");
throw err;
})
.then(([buildSnapshotValid, resolveValid]) => {
logger.timeEnd("check build dependencies");
if (buildSnapshotValid && resolveValid) {
logger.time("restore pack content");
return resolve(
cacheEntry.data().then(d => {
logger.timeEnd("restore pack content");
return d;
})
);
return cacheEntry.data().then(d => {
logger.timeEnd("restore pack content");
return d;
});
}
);
});
return undefined;
});
}
return cacheEntry;
})
@ -299,19 +375,14 @@ class PackFileCacheStrategy {
return new Pack(version, logger);
}
this.buildSnapshot = buildSnapshot;
this.oldBuildDependencies = buildDependencies;
if (buildDependencies) this.buildDependencies = buildDependencies;
if (newBuildDependencies)
this.newBuildDependencies.addAll(newBuildDependencies);
this.resolveResults = resolveResults;
this.resolveBuildDependenciesSnapshot = resolveBuildDependenciesSnapshot;
return cacheEntry;
}
return new Pack(version, logger);
})
.catch(err => {
if (err && err.code !== "ENOENT") {
logger.warn(
`Restoring pack failed from ${cacheLocation}.pack: ${err}`
);
logger.debug(err.stack);
}
return new Pack(version, logger);
});
}
@ -336,7 +407,7 @@ class PackFileCacheStrategy {
}
storeBuildDependencies(dependencies) {
this.buildDependencies.addAll(dependencies);
this.newBuildDependencies.addAll(dependencies);
}
afterAllStored() {
@ -344,13 +415,13 @@ class PackFileCacheStrategy {
if (!pack.invalid) return;
let promise;
const newBuildDependencies = new Set();
for (const dep of this.buildDependencies) {
if (!this.oldBuildDependencies.has(dep)) {
for (const dep of this.newBuildDependencies) {
if (!this.buildDependencies.has(dep)) {
newBuildDependencies.add(dep);
this.oldBuildDependencies.add(dep);
this.buildDependencies.add(dep);
}
}
this.buildDependencies.clear();
this.newBuildDependencies.clear();
if (newBuildDependencies.size > 0) {
this.logger.debug(
`Capturing build dependencies... (${Array.from(
@ -367,28 +438,62 @@ class PackFileCacheStrategy {
this.logger.timeEnd("resolve build dependencies");
this.logger.time("snapshot build dependencies");
const { files, directories, missing } = result;
this.fileSystemInfo.createSnapshot(
undefined,
const {
files,
directories,
missing,
{ hash: true },
resolveResults,
resolveDependencies
} = result;
if (this.resolveResults) {
for (const [key, value] of resolveResults) {
this.resolveResults.set(key, value);
}
} else {
this.resolveResults = resolveResults;
}
this.fileSystemInfo.createSnapshot(
undefined,
resolveDependencies.files,
resolveDependencies.directories,
resolveDependencies.missing,
{},
(err, snapshot) => {
if (err) return reject(err);
this.logger.timeEnd("snapshot build dependencies");
this.logger.debug("Captured build dependencies");
if (this.buildSnapshot) {
this.buildSnapshot = this.fileSystemInfo.mergeSnapshots(
this.buildSnapshot,
if (err) {
this.logger.timeEnd("snapshot build dependencies");
return reject(err);
}
if (this.resolveBuildDependenciesSnapshot) {
this.resolveBuildDependenciesSnapshot = this.fileSystemInfo.mergeSnapshots(
this.resolveBuildDependenciesSnapshot,
snapshot
);
} else {
this.buildSnapshot = snapshot;
this.resolveBuildDependenciesSnapshot = snapshot;
}
this.fileSystemInfo.createSnapshot(
undefined,
files,
directories,
missing,
{ hash: true },
(err, snapshot) => {
this.logger.timeEnd("snapshot build dependencies");
if (err) return reject(err);
this.logger.debug("Captured build dependencies");
resolve();
if (this.buildSnapshot) {
this.buildSnapshot = this.fileSystemInfo.mergeSnapshots(
this.buildSnapshot,
snapshot
);
} else {
this.buildSnapshot = snapshot;
}
resolve();
}
);
}
);
}
@ -405,7 +510,9 @@ class PackFileCacheStrategy {
? new DataWithBuildSnapshot(
() => pack,
this.buildSnapshot,
this.oldBuildDependencies
this.buildDependencies,
this.resolveResults,
this.resolveBuildDependenciesSnapshot
)
: pack;
// You might think this breaks all access to the existing pack

View File

@ -11,16 +11,16 @@
"@webassemblyjs/wasm-parser": "1.8.5",
"acorn": "^7.0.0",
"chrome-trace-event": "^1.0.2",
"enhanced-resolve": "5.0.0-beta.2",
"enhanced-resolve": "5.0.0-beta.3",
"eslint-scope": "^5.0.0",
"events": "^3.0.0",
"find-cache-dir": "^3.0.0",
"glob-to-regexp": "^0.4.1",
"graceful-fs": "^4.1.15",
"json-parse-better-errors": "^1.0.2",
"loader-runner": "3.0.0",
"loader-utils": "^1.2.3",
"neo-async": "^2.6.1",
"pkg-dir": "^4.2.0",
"schema-utils": "^2.1.0",
"tapable": "2.0.0-beta.8",
"terser-webpack-plugin": "^1.4.1",

View File

@ -1786,10 +1786,10 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0:
dependencies:
once "^1.4.0"
enhanced-resolve@5.0.0-beta.2:
version "5.0.0-beta.2"
resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.0.0-beta.2.tgz#9dfd00b7d73e7d6e0acab9e1f54241a557edbe8d"
integrity sha512-qo97XNV7JrYZR9eaFRPfKtOtiF1RBTGIx9v+j5DttaZyAhwgXnhs69YYTzEKgQpiOW8snnAiVdp5/eu+d3/55g==
enhanced-resolve@5.0.0-beta.3:
version "5.0.0-beta.3"
resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.0.0-beta.3.tgz#1f5b24d223db90a2e86235c365e337fcbf28a68b"
integrity sha512-SMjrP/h0qau9mj6L+dY0I350vTnHUDNm3hjV2QRzqhfaa6UYAkDoLfuRWPqAErKDejveFPyGLiQx1A8a5uR5hQ==
dependencies:
graceful-fs "^4.2.0"
tapable "^2.0.0-beta.8"
@ -2347,15 +2347,6 @@ find-cache-dir@^2.1.0:
make-dir "^2.0.0"
pkg-dir "^3.0.0"
find-cache-dir@^3.0.0:
version "3.0.0"
resolved "https://registry.yarnpkg.com/find-cache-dir/-/find-cache-dir-3.0.0.tgz#cd4b7dd97b7185b7e17dbfe2d6e4115ee3eeb8fc"
integrity sha512-t7ulV1fmbxh5G9l/492O1p5+EBbr3uwpt6odhFTMc+nWyhmbloe+ja9BZ8pIBtqFWhOmCWVjx+pTW4zDkFoclw==
dependencies:
commondir "^1.0.1"
make-dir "^3.0.0"
pkg-dir "^4.1.0"
find-up@^2.0.0:
version "2.1.0"
resolved "https://registry.yarnpkg.com/find-up/-/find-up-2.1.0.tgz#45d1b7e506c717ddd482775a2b77920a3c0c57a7"
@ -4920,7 +4911,7 @@ pkg-dir@^3.0.0:
dependencies:
find-up "^3.0.0"
pkg-dir@^4.1.0, pkg-dir@^4.2.0:
pkg-dir@^4.2.0:
version "4.2.0"
resolved "https://registry.yarnpkg.com/pkg-dir/-/pkg-dir-4.2.0.tgz#f099133df7ede422e81d1d8448270eeb3e4261f3"
integrity sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==