Best effort to capture resolving of file build dependencies

This commit is contained in:
Tobias Koppers 2019-08-20 13:50:14 +02:00
parent 82452939f5
commit 41de897916
2 changed files with 217 additions and 172 deletions

View File

@ -9,7 +9,7 @@ const resolve = require("enhanced-resolve");
const asyncLib = require("neo-async");
const AsyncQueue = require("./util/AsyncQueue");
const createHash = require("./util/createHash");
const { join, dirname } = require("./util/fs");
const { join, dirname, relative } = require("./util/fs");
/** @typedef {import("./WebpackError")} WebpackError */
/** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
@ -22,10 +22,11 @@ let FS_ACCURACY = 2000;
const RBDT_RESOLVE = 0;
const RBDT_RESOLVE_DIRECTORY = 1;
const RBDT_DIRECTORY = 2;
const RBDT_FILE = 3;
const RBDT_DIRECTORY_DEPENDENCIES = 4;
const RBDT_FILE_DEPENDENCIES = 5;
const RBDT_RESOLVE_FILE = 2;
const RBDT_DIRECTORY = 3;
const RBDT_FILE = 4;
const RBDT_DIRECTORY_DEPENDENCIES = 5;
const RBDT_FILE_DEPENDENCIES = 6;
const INVALID = Symbol("invalid");
@ -229,190 +230,234 @@ class FileSystemInfo {
const resolveDirectories = new Set();
const resolveMissing = new Set();
const resolveResults = new Map();
/** @type {asyncLib.QueueObject<{type: number, path: string, context?: string }, Error>} */
const queue = asyncLib.queue(({ type, context, path }, callback) => {
const resolveDirectory = path => {
const key = `d\n${context}\n${path}`;
if (resolveResults.has(key)) {
return callback();
}
resolveContext(
context,
path,
{
fileDependencies: resolveFiles,
contextDependencies: resolveDirectories,
missingDependencies: resolveMissing
},
(err, result) => {
if (err) {
if (err.code === "ENOENT" || err.code === "UNDECLARED_DEPENDENCY")
return callback();
return callback(err);
}
resolveResults.set(key, result);
queue.push({
type: RBDT_DIRECTORY,
path: result
});
callback();
/** @type {asyncLib.QueueObject<{type: number, path: string, context?: string, expected?: string }, Error>} */
const queue = asyncLib.queue(
({ type, context, path, expected }, callback) => {
const resolveDirectory = path => {
const key = `d\n${context}\n${path}`;
if (resolveResults.has(key)) {
return callback();
}
);
};
const resolveFile = path => {
const key = `f\n${context}\n${path}`;
if (resolveResults.has(key)) {
return callback();
}
resolve(
context,
path,
{
fileDependencies: resolveFiles,
contextDependencies: resolveDirectories,
missingDependencies: resolveMissing
},
(err, result) => {
if (err) {
if (err.code === "ENOENT" || err.code === "UNDECLARED_DEPENDENCY")
return callback();
return callback(err);
}
resolveResults.set(key, result);
queue.push({
type: RBDT_FILE,
path: result
});
callback();
}
);
};
switch (type) {
case RBDT_RESOLVE: {
const isDirectory = /[\\/]$/.test(path);
if (isDirectory) {
resolveDirectory(path.slice(0, path.length - 1));
} else {
resolveFile(path);
}
break;
}
case RBDT_RESOLVE_DIRECTORY: {
resolveDirectory(path);
break;
}
case RBDT_FILE: {
if (files.has(path)) {
callback();
break;
}
this.fs.realpath(path, (err, realPath) => {
if (err) return callback(err);
if (realPath !== path) {
resolveFiles.add(path);
}
if (!files.has(realPath)) {
files.add(realPath);
resolveContext(
context,
path,
{
fileDependencies: resolveFiles,
contextDependencies: resolveDirectories,
missingDependencies: resolveMissing
},
(err, result) => {
if (err) {
if (
err.code === "ENOENT" ||
err.code === "UNDECLARED_DEPENDENCY"
)
return callback();
return callback(err);
}
resolveResults.set(key, result);
queue.push({
type: RBDT_FILE_DEPENDENCIES,
path: realPath
type: RBDT_DIRECTORY,
path: result
});
callback();
}
callback();
});
break;
}
case RBDT_DIRECTORY: {
if (directories.has(path)) {
callback();
break;
);
};
const resolveFile = path => {
const key = `f\n${context}\n${path}`;
if (resolveResults.has(key)) {
return callback();
}
this.fs.realpath(path, (err, realPath) => {
if (err) return callback(err);
if (realPath !== path) {
resolveFiles.add(path);
}
if (!directories.has(realPath)) {
directories.add(realPath);
queue.push({
type: RBDT_DIRECTORY_DEPENDENCIES,
path: realPath
});
}
callback();
});
break;
}
case RBDT_FILE_DEPENDENCIES: {
/** @type {NodeModule} */
const module = require.cache[path];
if (module && Array.isArray(module.children)) {
for (const child of module.children) {
if (child.id) {
resolve(
context,
path,
{
fileDependencies: resolveFiles,
contextDependencies: resolveDirectories,
missingDependencies: resolveMissing
},
(err, result) => {
if (expected) {
if (result === expected) {
resolveResults.set(key, result);
}
} else {
if (err) {
if (
err.code === "ENOENT" ||
err.code === "UNDECLARED_DEPENDENCY"
)
return callback();
return callback(err);
}
resolveResults.set(key, result);
queue.push({
type: RBDT_FILE,
path: child.id
path: result
});
}
callback();
}
} else {
// Unable to get dependencies from module system
// This may be because of an incomplete require.cache implementation like in jest
// Assume requires stay in directory and add the whole directory
const directory = dirname(this.fs, path);
queue.push({
type: RBDT_DIRECTORY,
path: directory
});
}
callback();
break;
}
case RBDT_DIRECTORY_DEPENDENCIES: {
const match = /(^.+[\\/]node_modules[\\/](?:@[^\\/]+[\\/])?[^\\/]+)/.exec(
path
);
const packagePath = match ? match[1] : path;
const packageJson = join(this.fs, packagePath, "package.json");
this.fs.readFile(packageJson, (err, content) => {
if (err) {
if (err.code === "ENOENT") {
resolveMissing.add(packageJson);
const parent = dirname(this.fs, packagePath);
if (parent !== packagePath) {
};
switch (type) {
case RBDT_RESOLVE: {
const isDirectory = /[\\/]$/.test(path);
if (isDirectory) {
resolveDirectory(path.slice(0, path.length - 1));
} else {
resolveFile(path);
}
break;
}
case RBDT_RESOLVE_DIRECTORY: {
resolveDirectory(path);
break;
}
case RBDT_RESOLVE_FILE: {
resolveFile(path);
break;
}
case RBDT_FILE: {
if (files.has(path)) {
callback();
break;
}
this.fs.realpath(path, (err, realPath) => {
if (err) return callback(err);
if (realPath !== path) {
resolveFiles.add(path);
}
if (!files.has(realPath)) {
files.add(realPath);
queue.push({
type: RBDT_FILE_DEPENDENCIES,
path: realPath
});
}
callback();
});
break;
}
case RBDT_DIRECTORY: {
if (directories.has(path)) {
callback();
break;
}
this.fs.realpath(path, (err, realPath) => {
if (err) return callback(err);
if (realPath !== path) {
resolveFiles.add(path);
}
if (!directories.has(realPath)) {
directories.add(realPath);
queue.push({
type: RBDT_DIRECTORY_DEPENDENCIES,
path: realPath
});
}
callback();
});
break;
}
case RBDT_FILE_DEPENDENCIES: {
/** @type {NodeModule} */
const module = require.cache[path];
if (module && Array.isArray(module.children)) {
children: for (const child of module.children) {
let childPath = child.filename;
if (childPath) {
queue.push({
type: RBDT_DIRECTORY_DEPENDENCIES,
path: parent
type: RBDT_FILE,
path: childPath
});
if (childPath.endsWith(".js"))
childPath = childPath.slice(0, -3);
const context = dirname(this.fs, path);
for (const modulePath of module.paths) {
if (childPath.startsWith(modulePath)) {
const request = childPath.slice(modulePath.length + 1);
queue.push({
type: RBDT_RESOLVE_FILE,
context,
path: request,
expected: childPath
});
continue children;
}
}
let request = relative(this.fs, context, childPath);
request = request.replace(/\\/g, "/");
if (!request.startsWith("../")) request = `./${request}`;
queue.push({
type: RBDT_RESOLVE_FILE,
context,
path: request,
expected: child.filename
});
}
callback();
return;
}
return callback(err);
}
resolveFiles.add(packageJson);
let packageData;
try {
packageData = JSON.parse(content.toString("utf-8"));
} catch (e) {
return callback(e);
}
const depsObject = packageData.dependencies;
if (typeof depsObject === "object" && depsObject) {
for (const dep of Object.keys(depsObject)) {
queue.push({
type: RBDT_RESOLVE_DIRECTORY,
context: packagePath,
path: dep
});
}
} else {
// Unable to get dependencies from module system
// This may be because of an incomplete require.cache implementation like in jest
// Assume requires stay in directory and add the whole directory
const directory = dirname(this.fs, path);
queue.push({
type: RBDT_DIRECTORY,
path: directory
});
}
callback();
});
break;
break;
}
case RBDT_DIRECTORY_DEPENDENCIES: {
const match = /(^.+[\\/]node_modules[\\/](?:@[^\\/]+[\\/])?[^\\/]+)/.exec(
path
);
const packagePath = match ? match[1] : path;
const packageJson = join(this.fs, packagePath, "package.json");
this.fs.readFile(packageJson, (err, content) => {
if (err) {
if (err.code === "ENOENT") {
resolveMissing.add(packageJson);
const parent = dirname(this.fs, packagePath);
if (parent !== packagePath) {
queue.push({
type: RBDT_DIRECTORY_DEPENDENCIES,
path: parent
});
}
callback();
return;
}
return callback(err);
}
resolveFiles.add(packageJson);
let packageData;
try {
packageData = JSON.parse(content.toString("utf-8"));
} catch (e) {
return callback(e);
}
const depsObject = packageData.dependencies;
if (typeof depsObject === "object" && depsObject) {
for (const dep of Object.keys(depsObject)) {
queue.push({
type: RBDT_RESOLVE_DIRECTORY,
context: packagePath,
path: dep
});
}
}
callback();
});
break;
}
}
}
}, 50);
},
50
);
queue.drain = () => {
callback(null, {
files,

View File

@ -62,7 +62,7 @@ describe("BuildDependencies", () => {
await exec("1");
await exec("2");
fs.writeFileSync(
path.resolve(inputDirectory, "config-dependency.js"),
path.resolve(inputDirectory, "config-dependency"),
"module.exports = Date.now();"
);
const now2 = Date.now();