refactor filesystem interfaces

add Compiler.intermediateFileSystem
avoid using `path` and `fs` module when possible
move `join`, `mkdirp` and `dirname` into utils
join and dirname is optional in FileSystem interface
remove mkdirp from Filesystem interface
This commit is contained in:
Tobias Koppers 2019-06-11 13:09:42 +02:00
parent 224daec09d
commit e9c0d068dd
39 changed files with 450 additions and 184 deletions

View File

@ -61,7 +61,7 @@ const cli = {
if (!cli.installed) {
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const readLine = require("readline");
const notify =

View File

@ -7,7 +7,6 @@
const parseJson = require("json-parse-better-errors");
const asyncLib = require("neo-async");
const path = require("path");
const {
SyncHook,
SyncBailHook,
@ -25,6 +24,7 @@ const RequestShortener = require("./RequestShortener");
const ResolverFactory = require("./ResolverFactory");
const Stats = require("./Stats");
const Watching = require("./Watching");
const { join, dirname, mkdirp } = require("./util/fs");
const { makePathsRelative } = require("./util/identifier");
/** @typedef {import("webpack-sources").Source} Source */
@ -36,6 +36,9 @@ const { makePathsRelative } = require("./util/identifier");
/** @typedef {import("./Chunk")} Chunk */
/** @typedef {import("./FileSystemInfo").FileSystemInfoEntry} FileSystemInfoEntry */
/** @typedef {import("./Module")} Module */
/** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
/** @typedef {import("./util/fs").IntermediateFileSystem} IntermediateFileSystem */
/** @typedef {import("./util/fs").OutputFileSystem} OutputFileSystem */
/**
* @typedef {Object} CompilationParams
@ -151,7 +154,11 @@ class Compiler {
/** @type {string} */
this.outputPath = "";
/** @type {OutputFileSystem} */
this.outputFileSystem = null;
/** @type {IntermediateFileSystem} */
this.intermediateFileSystem = null;
/** @type {InputFileSystem} */
this.inputFileSystem = null;
this.watchFileSystem = null;
@ -353,7 +360,8 @@ class Compiler {
const writeOut = err => {
if (err) return callback(err);
const targetPath = this.outputFileSystem.join(
const targetPath = join(
this.outputFileSystem,
outputPath,
targetFile
);
@ -424,11 +432,9 @@ class Compiler {
};
if (targetFile.match(/\/|\\/)) {
const dir = path.dirname(targetFile);
this.outputFileSystem.mkdirp(
this.outputFileSystem.join(outputPath, dir),
writeOut
);
const fs = this.outputFileSystem;
const dir = dirname(fs, join(fs, outputPath, targetFile));
mkdirp(fs, dir, writeOut);
} else {
writeOut();
}
@ -448,7 +454,7 @@ class Compiler {
this.hooks.emit.callAsync(compilation, err => {
if (err) return callback(err);
outputPath = compilation.getPath(this.outputPath, {});
this.outputFileSystem.mkdirp(outputPath, emitFiles);
mkdirp(this.outputFileSystem, outputPath, emitFiles);
});
}
@ -458,14 +464,6 @@ class Compiler {
*/
emitRecords(callback) {
if (!this.recordsOutputPath) return callback();
const idx1 = this.recordsOutputPath.lastIndexOf("/");
const idx2 = this.recordsOutputPath.lastIndexOf("\\");
let recordsOutputPathDirectory = null;
if (idx1 > idx2) {
recordsOutputPathDirectory = this.recordsOutputPath.substr(0, idx1);
} else if (idx1 < idx2) {
recordsOutputPathDirectory = this.recordsOutputPath.substr(0, idx2);
}
const writeFile = () => {
this.outputFileSystem.writeFile(
@ -491,10 +489,14 @@ class Compiler {
);
};
const recordsOutputPathDirectory = dirname(
this.outputFileSystem,
this.recordsOutputPath
);
if (!recordsOutputPathDirectory) {
return writeFile();
}
this.outputFileSystem.mkdirp(recordsOutputPathDirectory, err => {
mkdirp(this.outputFileSystem, recordsOutputPathDirectory, err => {
if (err) return callback(err);
writeFile();
});
@ -630,11 +632,12 @@ class Compiler {
}
createNormalModuleFactory() {
const normalModuleFactory = new NormalModuleFactory(
this.options.context,
this.resolverFactory,
this.options.module || {}
);
const normalModuleFactory = new NormalModuleFactory({
context: this.options.context,
fs: this.inputFileSystem,
resolverFactory: this.resolverFactory,
options: this.options.module || {}
});
this.hooks.normalModuleFactory.call(normalModuleFactory);
return normalModuleFactory;
}

View File

@ -6,12 +6,11 @@
"use strict";
const asyncLib = require("neo-async");
const path = require("path");
const { AsyncSeriesWaterfallHook, SyncWaterfallHook } = require("tapable");
const ContextModule = require("./ContextModule");
const ModuleFactory = require("./ModuleFactory");
const ContextElementDependency = require("./dependencies/ContextElementDependency");
const { join } = require("./util/fs");
/** @typedef {import("./Module")} Module */
/** @typedef {import("./ModuleFactory").ModuleFactoryCreateData} ModuleFactoryCreateData */
@ -195,7 +194,7 @@ module.exports = class ContextModuleFactory extends ModuleFactory {
asyncLib.map(
files.filter(p => p.indexOf(".") !== 0),
(segment, callback) => {
const subResource = path.join(directory, segment);
const subResource = join(fs, directory, segment);
if (!exclude || !subResource.match(exclude)) {
fs.stat(subResource, (err, stat) => {

View File

@ -5,8 +5,8 @@
"use strict";
const path = require("path");
const ContextElementDependency = require("./dependencies/ContextElementDependency");
const { join } = require("./util/fs");
class ContextReplacementPlugin {
constructor(
@ -84,7 +84,18 @@ class ContextReplacementPlugin {
if (!result) return;
if (resourceRegExp.test(result.resource)) {
if (newContentResource !== undefined) {
result.resource = path.resolve(result.resource, newContentResource);
if (
newContentResource.startsWith("/") ||
(newContentResource.length > 1 && newContentResource[1] === ":")
) {
result.resource = newContentResource;
} else {
result.resource = join(
compiler.inputFileSystem,
result.resource,
newContentResource
);
}
}
if (newContentRecursive !== undefined) {
result.recursive = newContentRecursive;
@ -100,8 +111,17 @@ class ContextReplacementPlugin {
if (typeof newContentCallback === "function") {
const origResource = result.resource;
newContentCallback(result);
if (result.resource !== origResource) {
result.resource = path.resolve(origResource, result.resource);
if (
result.resource !== origResource &&
!result.resource.startsWith("/") &&
(result.resource.length <= 1 || result.resource[1] !== ":")
) {
// When the function changed it to an relative path
result.resource = join(
compiler.inputFileSystem,
origResource,
result.resource
);
}
} else {
for (const d of result.dependencies) {

View File

@ -5,20 +5,10 @@
"use strict";
const path = require("path");
const BasicEvaluatedExpression = require("./BasicEvaluatedExpression");
const UnsupportedFeatureWarning = require("./UnsupportedFeatureWarning");
const ConstDependency = require("./dependencies/ConstDependency");
exports.getModulePath = (context, pathToModule) => {
let moduleJsPath = path.relative(context, pathToModule);
if (!/^[A-Z]:/i.test(moduleJsPath)) {
moduleJsPath = "./" + moduleJsPath.replace(/\\/g, "/");
}
return moduleJsPath;
};
exports.toConstantDependency = (parser, value, runtimeRequirements) => {
return function constDependency(expr) {
const dep = new ConstDependency(value, expr.range, runtimeRequirements);

View File

@ -6,9 +6,9 @@
"use strict";
const asyncLib = require("neo-async");
const path = require("path");
const EntryDependency = require("./dependencies/EntryDependency");
const { compareModulesById } = require("./util/comparators");
const { dirname, mkdirp } = require("./util/fs");
/** @typedef {import("./Compiler")} Compiler */
@ -99,14 +99,18 @@ class LibManifestPlugin {
? JSON.stringify(manifest, null, 2)
: JSON.stringify(manifest);
const content = Buffer.from(manifestContent, "utf8");
compiler.outputFileSystem.mkdirp(path.dirname(targetPath), err => {
if (err) return callback(err);
compiler.outputFileSystem.writeFile(
targetPath,
content,
callback
);
});
mkdirp(
compiler.intermediateFileSystem,
dirname(compiler.intermediateFileSystem, targetPath),
err => {
if (err) return callback(err);
compiler.intermediateFileSystem.writeFile(
targetPath,
content,
callback
);
}
);
},
callback
);

View File

@ -17,6 +17,9 @@ const MultiWatching = require("./MultiWatching");
/** @typedef {import("./Compiler")} Compiler */
/** @typedef {import("./Stats")} Stats */
/** @typedef {import("./Watching")} Watching */
/** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
/** @typedef {import("./util/fs").IntermediateFileSystem} IntermediateFileSystem */
/** @typedef {import("./util/fs").OutputFileSystem} OutputFileSystem */
/** @typedef {number} CompilerStatus */
@ -117,18 +120,37 @@ module.exports = class MultiCompiler {
throw new Error("Cannot read outputFileSystem of a MultiCompiler");
}
get intermediateFileSystem() {
throw new Error("Cannot read outputFileSystem of a MultiCompiler");
}
/**
* @param {InputFileSystem} value the new input file system
*/
set inputFileSystem(value) {
for (const compiler of this.compilers) {
compiler.inputFileSystem = value;
}
}
/**
* @param {OutputFileSystem} value the new output file system
*/
set outputFileSystem(value) {
for (const compiler of this.compilers) {
compiler.outputFileSystem = value;
}
}
/**
* @param {IntermediateFileSystem} value the new intermediate file system
*/
set intermediateFileSystem(value) {
for (const compiler of this.compilers) {
compiler.intermediateFileSystem = value;
}
}
/**
* @param {Compiler} compiler the child compiler
* @param {string[]} dependencies its dependencies

View File

@ -5,12 +5,12 @@
"use strict";
const path = require("path");
const {
evaluateToString,
expressionIsUnsupported
} = require("./JavascriptParserHelpers");
const CachedConstDependency = require("./dependencies/CachedConstDependency");
const { relative } = require("./util/fs");
/** @typedef {import("webpack-sources").ReplaceSource} ReplaceSource */
/** @typedef {import("./Compiler")} Compiler */
@ -65,7 +65,7 @@ class NodeStuffPlugin {
setConstant("__filename", "/index.js");
} else {
setModuleConstant("__filename", module =>
path.relative(context, module.resource)
relative(compiler.inputFileSystem, context, module.resource)
);
}
parser.hooks.evaluateIdentifier
@ -84,7 +84,7 @@ class NodeStuffPlugin {
setConstant("__dirname", "/");
} else {
setModuleConstant("__dirname", module =>
path.relative(context, module.context)
relative(compiler.inputFileSystem, context, module.context)
);
}
parser.hooks.evaluateIdentifier

View File

@ -6,7 +6,6 @@
"use strict";
const asyncLib = require("neo-async");
const path = require("path");
const {
AsyncSeriesBailHook,
SyncWaterfallHook,
@ -23,6 +22,7 @@ const BasicMatcherRulePlugin = require("./rules/BasicMatcherRulePlugin");
const RuleSetCompiler = require("./rules/RuleSetCompiler");
const UseEffectRulePlugin = require("./rules/UseEffectRulePlugin");
const { cachedCleverMerge } = require("./util/cleverMerge");
const { join } = require("./util/fs");
/** @typedef {import("./ModuleFactory").ModuleFactoryCreateData} ModuleFactoryCreateData */
/** @typedef {import("./ModuleFactory").ModuleFactoryResult} ModuleFactoryResult */
@ -123,7 +123,7 @@ const ruleSetCompiler = new RuleSetCompiler([
]);
class NormalModuleFactory extends ModuleFactory {
constructor(context, resolverFactory, options) {
constructor({ context, fs, resolverFactory, options }) {
super();
this.hooks = Object.freeze({
/** @type {AsyncSeriesBailHook<[ResolveData], TODO>} */
@ -162,6 +162,7 @@ class NormalModuleFactory extends ModuleFactory {
? options.unsafeCache
: () => true;
this.context = context || "";
this.fs = fs;
this.parserCache = Object.create(null);
this.generatorCache = Object.create(null);
this.hooks.factorize.tapAsync(
@ -250,7 +251,7 @@ class NormalModuleFactory extends ModuleFactory {
(secondChar === 46 && matchResource.charCodeAt(2) === 47)
) {
// if matchResources startsWith ../ or ./
matchResource = path.join(context, matchResource);
matchResource = join(this.fs, context, matchResource);
}
}
requestWithoutMatchResource = request.substr(

View File

@ -5,7 +5,7 @@
"use strict";
const path = require("path");
const { join, dirname } = require("./util/fs");
/** @typedef {import("./Compiler")} Compiler */
/** @typedef {function(TODO): void} ModuleReplacer */
@ -48,10 +48,19 @@ class NormalModuleReplacementPlugin {
if (typeof newResource === "function") {
newResource(result);
} else {
createData.resource = path.resolve(
path.dirname(createData.resource),
newResource
);
const fs = compiler.inputFileSystem;
if (
newResource.startsWith("/") ||
(newResource.length > 1 && newResource[1] === ":")
) {
createData.resource = newResource;
} else {
createData.resource = join(
fs,
dirname(fs, createData.resource),
newResource
);
}
}
}
return result;

View File

@ -5,7 +5,7 @@
"use strict";
const path = require("path");
const { join, dirname: fsDirname } = require("./util/fs");
const NORMALIZE_SLASH_DIRECTION_REGEXP = /\\/g;
const PATH_CHARS_REGEXP = /[-[\]{}()*+?.,\\^$|#\s]/g;
@ -54,7 +54,7 @@ class RequestShortener {
this.currentDirectoryRegExp = createRegExpForPath(directory);
}
const dirname = path.dirname(directory);
const dirname = fsDirname(undefined, directory);
const endsWithSeparator = SEPARATOR_REGEXP.test(dirname);
const parentDirectory = endsWithSeparator
? dirname.substr(0, dirname.length - 1)
@ -65,7 +65,9 @@ class RequestShortener {
}
if (__dirname.length >= 2) {
const buildins = normalizeBackSlashDirection(path.join(__dirname, ".."));
const buildins = normalizeBackSlashDirection(
join(undefined, __dirname, "..")
);
const buildinsAsModule =
this.currentDirectoryRegExp &&
this.currentDirectoryRegExp.test(buildins);

View File

@ -5,13 +5,13 @@
"use strict";
const path = require("path");
const validateOptions = require("schema-utils");
const { ConcatSource, RawSource } = require("webpack-sources");
const ModuleFilenameHelpers = require("./ModuleFilenameHelpers");
const ProgressPlugin = require("./ProgressPlugin");
const SourceMapDevToolModuleOptionsPlugin = require("./SourceMapDevToolModuleOptionsPlugin");
const createHash = require("./util/createHash");
const { relative, dirname } = require("./util/fs");
const schema = require("../schemas/plugins/SourceMapDevToolPlugin.json");
@ -80,6 +80,7 @@ class SourceMapDevToolPlugin {
* @returns {void}
*/
apply(compiler) {
const outputFs = compiler.outputFileSystem;
const sourceMapFilename = this.sourceMapFilename;
const sourceMappingURLComment = this.sourceMappingURLComment;
const moduleFilenameTemplate = this.moduleFilenameTemplate;
@ -249,17 +250,23 @@ class SourceMapDevToolPlugin {
let sourceMapFile = compilation.getPath(sourceMapFilename, {
chunk,
filename: options.fileContext
? path.relative(options.fileContext, filename)
? relative(
outputFs,
`/${options.fileContext}`,
`/${filename}`
)
: filename,
contentHash: createHash("md4")
.update(sourceMapString)
.digest("hex")
});
const sourceMapUrl = options.publicPath
? options.publicPath + sourceMapFile.replace(/\\/g, "/")
: path
.relative(path.dirname(file), sourceMapFile)
.replace(/\\/g, "/");
? options.publicPath + sourceMapFile
: relative(
outputFs,
dirname(outputFs, `/${file}`),
`/${sourceMapFile}`
);
if (currentSourceMappingURLComment !== false) {
assets[file] = compilation.assets[file] = new ConcatSource(
new RawSource(source),

View File

@ -550,6 +550,7 @@ class WebpackOptionsApply extends OptionsApply {
const SeparateFilesCacheStrategy = require("./cache/SeparateFilesCacheStrategy");
new InstantFileCachePlugin(
new SeparateFilesCacheStrategy({
fs: compiler.intermediateFileSystem,
cacheLocation: cacheOptions.cacheLocation,
version: cacheOptions.version,
hashAlgorithm: cacheOptions.hashAlgorithm,
@ -563,6 +564,7 @@ class WebpackOptionsApply extends OptionsApply {
const SeparateFilesCacheStrategy = require("./cache/SeparateFilesCacheStrategy");
new BackgroundFileCachePlugin(
new SeparateFilesCacheStrategy({
fs: compiler.intermediateFileSystem,
cacheLocation: cacheOptions.cacheLocation,
version: cacheOptions.version,
hashAlgorithm: cacheOptions.hashAlgorithm,
@ -576,6 +578,7 @@ class WebpackOptionsApply extends OptionsApply {
const SeparateFilesCacheStrategy = require("./cache/SeparateFilesCacheStrategy");
new IdleFileCachePlugin(
new SeparateFilesCacheStrategy({
fs: compiler.intermediateFileSystem,
cacheLocation: cacheOptions.cacheLocation,
version: cacheOptions.version,
hashAlgorithm: cacheOptions.hashAlgorithm,
@ -589,6 +592,7 @@ class WebpackOptionsApply extends OptionsApply {
const PackFileCacheStrategy = require("./cache/PackFileCacheStrategy");
new IdleFileCachePlugin(
new PackFileCacheStrategy({
fs: compiler.intermediateFileSystem,
cacheLocation: cacheOptions.cacheLocation,
version: cacheOptions.version,
loglevel: cacheOptions.loglevel

View File

@ -7,7 +7,7 @@
const makeSerializable = require("../util/makeSerializable");
const {
fileSerializer,
createFileSerializer,
NOT_SERIALIZABLE,
MEASURE_START_OPERATION,
MEASURE_END_OPERATION
@ -202,13 +202,14 @@ makeSerializable(
);
class PackFileCacheStrategy {
constructor({ cacheLocation, version, loglevel }) {
constructor({ fs, cacheLocation, version, loglevel }) {
this.fileSerializer = createFileSerializer(fs);
this.cacheLocation = cacheLocation;
const log = loglevel
? { debug: 4, verbose: 3, info: 2, warning: 1 }[loglevel]
: 0;
this.log = log;
this.packPromise = fileSerializer
this.packPromise = this.fileSerializer
.deserialize({ filename: `${cacheLocation}.pack` })
.then(cacheEntry => {
if (cacheEntry) {
@ -279,7 +280,7 @@ class PackFileCacheStrategy {
// which are still referenced, but serializing the pack memorizes
// all data in the pack and makes it no longer need the backing file
// So it's safe to replace the pack file
return fileSerializer
return this.fileSerializer
.serialize(pack, {
filename: `${this.cacheLocation}.pack`
})

View File

@ -5,12 +5,14 @@
"use strict";
const path = require("path");
const createHash = require("../util/createHash");
const { fileSerializer } = require("../util/serialization");
const { join } = require("../util/fs");
const { createFileSerializer } = require("../util/serialization");
class SeparateFilesCacheStrategy {
constructor({ cacheLocation, version, hashAlgorithm, loglevel }) {
constructor({ fs, cacheLocation, version, hashAlgorithm, loglevel }) {
this.fs = fs;
this.fileSerializer = createFileSerializer(fs);
this.cacheLocation = cacheLocation;
this.version = version;
this.log = loglevel
@ -32,8 +34,8 @@ class SeparateFilesCacheStrategy {
version: this.version
};
const relativeFilename = this.toHash(identifier) + ".data";
const filename = path.join(this.cacheLocation, relativeFilename);
return fileSerializer
const filename = join(this.fs, this.cacheLocation, relativeFilename);
return this.fileSerializer
.serialize(entry, { filename })
.then(() => {
if (this.log >= 2) {
@ -53,8 +55,8 @@ class SeparateFilesCacheStrategy {
restore(identifier, etag) {
const relativeFilename = this.toHash(identifier) + ".data";
const filename = path.join(this.cacheLocation, relativeFilename);
return fileSerializer
const filename = join(this.fs, this.cacheLocation, relativeFilename);
return this.fileSerializer
.deserialize({ filename })
.then(cacheEntry => {
if (cacheEntry === undefined) {

View File

@ -5,13 +5,12 @@
"use strict";
const { Tracer } = require("chrome-trace-event");
const fs = require("fs");
const mkdirp = require("mkdirp");
const path = require("path");
const validateOptions = require("schema-utils");
const schema = require("../../schemas/plugins/debug/ProfilingPlugin.json");
const { dirname, mkdirpSync } = require("../util/fs");
/** @typedef {import("../../declarations/plugins/debug/ProfilingPlugin").ProfilingPluginOptions} ProfilingPluginOptions */
/** @typedef {import("../util/fs").IntermediateFileSystem} IntermediateFileSystem */
let inspector = undefined;
@ -93,17 +92,18 @@ class Profiler {
*/
/**
* @param {IntermediateFileSystem} fs filesystem used for output
* @param {string} outputPath The location where to write the log.
* @returns {Trace} The trace object
*/
const createTrace = outputPath => {
const createTrace = (fs, outputPath) => {
const trace = new Tracer({
noStream: true
});
const profiler = new Profiler(inspector);
if (/\/|\\/.test(outputPath)) {
const dirPath = path.dirname(outputPath);
mkdirp.sync(dirPath);
const dirPath = dirname(fs, outputPath);
mkdirpSync(fs, dirPath);
}
const fsStream = fs.createWriteStream(outputPath);
@ -170,7 +170,10 @@ class ProfilingPlugin {
}
apply(compiler) {
const tracer = createTrace(this.outputPath);
const tracer = createTrace(
compiler.intermediateFileSystem,
this.outputPath
);
tracer.profiler.startProfiling();
// Compiler Hooks

View File

@ -7,7 +7,7 @@
const CachedInputFileSystem = require("enhanced-resolve/lib/CachedInputFileSystem");
const NodeJsInputFileSystem = require("enhanced-resolve/lib/NodeJsInputFileSystem");
const NodeOutputFileSystem = require("./NodeOutputFileSystem");
const fs = require("graceful-fs");
const NodeWatchFileSystem = require("./NodeWatchFileSystem");
/** @typedef {import("../Compiler")} Compiler */
@ -23,7 +23,8 @@ class NodeEnvironmentPlugin {
60000
);
const inputFileSystem = compiler.inputFileSystem;
compiler.outputFileSystem = new NodeOutputFileSystem();
compiler.outputFileSystem = fs;
compiler.intermediateFileSystem = fs;
compiler.watchFileSystem = new NodeWatchFileSystem(
compiler.inputFileSystem
);

View File

@ -1,23 +0,0 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
const fs = require("fs");
const mkdirp = require("mkdirp");
const path = require("path");
class NodeOutputFileSystem {
constructor() {
this.mkdirp = mkdirp;
this.mkdir = fs.mkdir.bind(fs);
this.rmdir = fs.rmdir.bind(fs);
this.unlink = fs.unlink.bind(fs);
this.writeFile = fs.writeFile.bind(fs);
this.join = path.join.bind(path);
}
}
module.exports = NodeOutputFileSystem;

View File

@ -4,10 +4,8 @@
"use strict";
const fs = require("fs");
const mkdirp = require("mkdirp");
const path = require("path");
const Queue = require("../util/Queue");
const { dirname, mkdirp } = require("../util/fs");
const memorize = require("../util/memorize");
const SerializerMiddleware = require("./SerializerMiddleware");
@ -96,7 +94,8 @@ class Section {
*/
class FileManager {
constructor() {
constructor(fs) {
this.fs = fs;
/** @type {Map<string, Queue<FileJob>>} */
this.jobs = new Map();
this.processing = new Map();
@ -161,7 +160,7 @@ class FileManager {
* @returns {void}
*/
const closeFile = next => {
fs.close(fileHandle, err => {
this.fs.close(fileHandle, err => {
if (err) return handleError(err);
fileHandle = undefined;
next();
@ -174,7 +173,7 @@ class FileManager {
const openFile = () => {
if (fileHandle === undefined) {
write = currentJob.write;
fs.open(filename, write ? "w" : "r", (err, file) => {
this.fs.open(filename, write ? "w" : "r", (err, file) => {
if (err) return handleError(err);
fileHandle = file;
process();
@ -210,20 +209,37 @@ class FileManager {
next();
}
}
const fileManager = new FileManager();
const createPointer = (filename, offset, size) => {
const fileManagers = new WeakMap();
const getFileManager = fs => {
const fm = fileManagers.get(fs);
if (fm !== undefined) return fm;
const fileManager = new FileManager(fs);
fileManagers.set(fs, fileManager);
return fileManager;
};
const createPointer = (fs, fileManager, filename, offset, size) => {
return memorize(() => {
return new Promise((resolve, reject) => {
fileManager.addJob(
filename,
false,
(file, callback) => {
readSection(filename, file, offset, size, (err, parts) => {
if (err) return callback(err);
resolve(parts);
callback();
});
readSection(
fs,
fileManager,
filename,
file,
offset,
size,
(err, parts) => {
if (err) return callback(err);
resolve(parts);
callback();
}
);
},
reject
);
@ -231,7 +247,14 @@ const createPointer = (filename, offset, size) => {
});
};
const readFileSectionToBuffer = (fd, buffer, offset, position, callback) => {
const readFileSectionToBuffer = (
fs,
fd,
buffer,
offset,
position,
callback
) => {
const remaining = buffer.length - offset;
fs.read(fd, buffer, offset, remaining, position, (err, bytesRead) => {
if (err) return callback(err);
@ -244,6 +267,7 @@ const readFileSectionToBuffer = (fd, buffer, offset, position, callback) => {
}
if (bytesRead < remaining) {
return readFileSectionToBuffer(
fs,
fd,
buffer,
offset + bytesRead,
@ -255,9 +279,17 @@ const readFileSectionToBuffer = (fd, buffer, offset, position, callback) => {
});
};
const readSection = (filename, file, offset, size, callback) => {
const readSection = (
fs,
fileManager,
filename,
file,
offset,
size,
callback
) => {
const buffer = Buffer.allocUnsafe(size);
readFileSectionToBuffer(file, buffer, 0, offset, err => {
readFileSectionToBuffer(fs, file, buffer, 0, offset, err => {
if (err) return callback(err);
const result = [];
@ -270,7 +302,7 @@ const readSection = (filename, file, offset, size, callback) => {
pos += 4;
const pSize = buffer.readUInt32LE(pos);
pos += 4;
result.push(createPointer(filename, pOffset, pSize));
result.push(createPointer(fs, fileManager, filename, pOffset, pSize));
} else {
const buf = buffer.slice(pos, pos + len);
pos += len;
@ -281,7 +313,7 @@ const readSection = (filename, file, offset, size, callback) => {
});
};
const writeBuffers = (fileHandle, buffers, callback) => {
const writeBuffers = (fs, fileHandle, buffers, callback) => {
const stream = fs.createWriteStream(null, {
fd: fileHandle,
autoClose: false
@ -309,10 +341,15 @@ const writeBuffers = (fileHandle, buffers, callback) => {
/**
* @typedef {BufferSerializableType[]} DeserializedType
* @typedef {undefined} SerializedType
* @typedef {true} SerializedType
* @extends {SerializerMiddleware<DeserializedType, SerializedType>}
*/
class FileMiddleware extends SerializerMiddleware {
constructor(fs) {
super();
this.fs = fs;
this.fileManager = getFileManager(fs);
}
/**
* @param {DeserializedType} data data
* @param {Object} context context object
@ -350,10 +387,10 @@ class FileMiddleware extends SerializerMiddleware {
// write to file
return new Promise((resolve, reject) => {
mkdirp(path.dirname(filename), err => {
mkdirp(this.fs, dirname(this.fs, filename), err => {
if (err) return reject(err);
fileManager.addJob(filename, true, (file, callback) => {
writeBuffers(file, buffers, err => {
this.fileManager.addJob(filename, true, (file, callback) => {
writeBuffers(this.fs, file, buffers, err => {
if (err) return callback(err);
resolve(true);
callback();
@ -371,22 +408,30 @@ class FileMiddleware extends SerializerMiddleware {
*/
deserialize(data, { filename }) {
return new Promise((resolve, reject) => {
fileManager.addJob(
this.fileManager.addJob(
filename,
false,
(file, callback) => {
const sizeBuf = Buffer.allocUnsafe(4);
readFileSectionToBuffer(file, sizeBuf, 0, 0, err => {
readFileSectionToBuffer(this.fs, file, sizeBuf, 0, 0, err => {
if (err) return callback(err);
const rootSize = sizeBuf.readUInt32LE(0);
readSection(filename, file, 4, rootSize, (err, parts) => {
if (err) return callback(err);
readSection(
this.fs,
this.fileManager,
filename,
file,
4,
rootSize,
(err, parts) => {
if (err) return callback(err);
resolve(parts);
callback();
});
resolve(parts);
callback();
}
);
});
},
reject

174
lib/util/fs.js Normal file
View File

@ -0,0 +1,174 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Tobias Koppers @sokra
*/
"use strict";
const path = require("path");
/** @typedef {function(NodeJS.ErrnoException=): void} Callback */
/** @typedef {function(NodeJS.ErrnoException=, Buffer=): void} BufferCallback */
/** @typedef {function(NodeJS.ErrnoException=, import("fs").Stats=): void} StatsCallback */
/**
* @typedef {Object} OutputFileSystem
* @property {function(string, Buffer|string, Callback): void} writeFile
* @property {function(string, Callback): void} mkdir
* @property {(function(string, string): string)=} join
* @property {(function(string, string): string)=} relative
* @property {(function(string): string)=} dirname
*/
/**
* @typedef {Object} IntermediateFileSystemExtras
* @property {function(string): void} mkdirSync
* @property {function(string): import("fs").WriteStream} createWriteStream
*/
/** @typedef {OutputFileSystem & IntermediateFileSystemExtras} IntermediateFileSystem */
/**
* @typedef {Object} InputFileSystem
* @property {function(string, BufferCallback): void} readFile
* @property {function(string, StatsCallback): void} stat
* @property {(function(string=): void)=} purge
* @property {(function(string, string): string)=} join
* @property {(function(string, string): string)=} relative
* @property {(function(string): string)=} dirname
*/
/**
*
* @param {(InputFileSystem|OutputFileSystem)=} fs a file system
* @param {string} rootPath the root path
* @param {string} targetPath the target path
* @returns {string} location of targetPath relative to rootPath
*/
const relative = (fs, rootPath, targetPath) => {
if (fs && fs.relative) {
return fs.relative(rootPath, targetPath);
} else if (rootPath.startsWith("/")) {
return path.posix.relative(rootPath, targetPath);
} else if (rootPath.length > 1 && rootPath[1] === ":") {
return path.win32.relative(rootPath, targetPath);
} else {
throw new Error(
`${rootPath} is neither a posix nor a windows path, and there is no 'relative' method defined in the file system`
);
}
};
exports.relative = relative;
/**
* @param {(InputFileSystem|OutputFileSystem)=} fs a file system
* @param {string} rootPath a path
* @param {string} filename a filename
* @returns {string} the joined path
*/
const join = (fs, rootPath, filename) => {
if (fs && fs.join) {
return fs.join(rootPath, filename);
} else if (rootPath.startsWith("/")) {
return path.posix.join(rootPath, filename);
} else if (rootPath.length > 1 && rootPath[1] === ":") {
return path.win32.join(rootPath, filename);
} else {
throw new Error(
`${rootPath} is neither a posix nor a windows path, and there is no 'join' method defined in the file system`
);
}
};
exports.join = join;
/**
* @param {(InputFileSystem|OutputFileSystem)=} fs a file system
* @param {string} absPath an absolute path
* @returns {string} the parent directory of the absolute path
*/
const dirname = (fs, absPath) => {
if (fs && fs.dirname) {
return fs.dirname(absPath);
} else if (absPath.startsWith("/")) {
return path.posix.dirname(absPath);
} else if (absPath.length > 1 && absPath[1] === ":") {
return path.win32.dirname(absPath);
} else {
throw new Error(
`${absPath} is neither a posix nor a windows path, and there is no 'dirname' method defined in the file system`
);
}
};
exports.dirname = dirname;
/**
* @param {OutputFileSystem} fs a file system
* @param {string} p an absolute path
* @param {function(Error=): void} callback callback function for the error
* @returns {void}
*/
const mkdirp = (fs, p, callback) => {
fs.mkdir(p, err => {
if (err) {
if (err.code === "ENOENT") {
const dir = dirname(fs, p);
if (dir === p) {
callback(err);
return;
}
mkdirp(fs, dir, err => {
if (err) {
callback(err);
return;
}
fs.mkdir(p, err => {
if (err) {
if (err.code === "EEXIST") {
callback();
return;
}
callback(err);
return;
}
callback();
});
});
return;
} else if (err.code === "EEXIST") {
callback();
return;
}
callback(err);
return;
}
callback();
});
};
exports.mkdirp = mkdirp;
/**
* @param {IntermediateFileSystem} fs a file system
* @param {string} p an absolute path
* @returns {void}
*/
const mkdirpSync = (fs, p) => {
try {
fs.mkdirSync(p);
} catch (err) {
if (err) {
if (err.code === "ENOENT") {
const dir = dirname(fs, p);
if (dir === p) {
throw err;
}
mkdirpSync(fs, dir);
fs.mkdirSync(p);
return;
} else if (err.code === "EEXIST") {
return;
}
throw err;
}
}
};
exports.mkdirpSync = mkdirpSync;

View File

@ -22,12 +22,13 @@ exports.registerNotSerializable = registerNotSerializable;
exports.NOT_SERIALIZABLE = ObjectMiddleware.NOT_SERIALIZABLE;
exports.MEASURE_START_OPERATION = BinaryMiddleware.MEASURE_START_OPERATION;
exports.MEASURE_END_OPERATION = BinaryMiddleware.MEASURE_END_OPERATION;
exports.fileSerializer = new Serializer([
new SingleItemMiddleware(),
new ObjectMiddleware(),
new BinaryMiddleware(),
new FileMiddleware()
]);
exports.createFileSerializer = fs =>
new Serializer([
new SingleItemMiddleware(),
new ObjectMiddleware(),
new BinaryMiddleware(),
new FileMiddleware(fs)
]);
exports.buffersSerializer = new Serializer([
new SingleItemMiddleware(),
new ObjectMiddleware(),

View File

@ -19,6 +19,7 @@
"events": "^3.0.0",
"find-cache-dir": "^2.1.0",
"glob-to-regexp": "^0.4.1",
"graceful-fs": "^4.1.15",
"json-parse-better-errors": "^1.0.2",
"loader-runner": "3.0.0",
"loader-utils": "^1.1.0",

View File

@ -1,7 +1,7 @@
"use strict";
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const asyncLib = require("neo-async");
const Benchmark = require("benchmark");
const { remove } = require("./helpers/remove");

View File

@ -2,7 +2,7 @@
"use strict";
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const rimraf = require("rimraf");
const webpack = require("..");
@ -24,19 +24,18 @@ describe("Compiler (caching)", () => {
options.output.filename = "bundle.js";
options.output.pathinfo = true;
const logs = {
mkdirp: [],
mkdir: [],
writeFile: []
};
const c = webpack(options);
const files = {};
c.outputFileSystem = {
join() {
return [].join.call(arguments, "/").replace(/\/+/g, "/");
},
mkdirp(path, callback) {
logs.mkdirp.push(path);
callback();
mkdir(path, callback) {
logs.mkdir.push(path);
const err = new Error();
err.code = "EEXIST";
callback(err);
},
writeFile(name, content, callback) {
logs.writeFile.push(name, content);

View File

@ -22,19 +22,18 @@ describe("Compiler", () => {
minimize: false
};
const logs = {
mkdirp: [],
mkdir: [],
writeFile: []
};
const c = webpack(options);
const files = {};
c.outputFileSystem = {
join() {
return [].join.call(arguments, "/").replace(/\/+/g, "/");
},
mkdirp(path, callback) {
logs.mkdirp.push(path);
callback();
mkdir(path, callback) {
logs.mkdir.push(path);
const err = new Error();
err.code = "EEXIST";
callback(err);
},
writeFile(name, content, callback) {
logs.writeFile.push(name, content);
@ -76,7 +75,7 @@ describe("Compiler", () => {
}
},
(stats, files) => {
expect(stats.logs.mkdirp).toEqual(["/what", "/what/the"]);
expect(stats.logs.mkdir).toEqual(["/what", "/what/the"]);
done();
}
);

View File

@ -2,7 +2,7 @@
/* globals describe expect it */
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const vm = require("vm");
const mkdirp = require("mkdirp");
const rimraf = require("rimraf");

View File

@ -1,4 +1,4 @@
const fs = require("fs");
const fs = require("graceful-fs");
const path = require("path");
const lockfile = require("@yarnpkg/lockfile");

View File

@ -2,7 +2,7 @@
/*globals describe it */
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const webpack = require("..");
const prettyFormat = require("pretty-format");
@ -81,8 +81,7 @@ const defaults = {
}
},
outputFileSystem: {
join: path.join.bind(path),
mkdirp(dir, callback) {
mkdir(dir, callback) {
callback();
},
writeFile(file, content, callback) {

View File

@ -2,7 +2,7 @@
/* globals describe it */
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const webpack = require("..");
describe("Examples", () => {

View File

@ -1,7 +1,7 @@
"use strict";
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const mkdirp = require("mkdirp");
const webpack = require("..");

View File

@ -2,7 +2,7 @@
/* globals expect */
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const vm = require("vm");
const checkArrayExpectation = require("./checkArrayExpectation");
const createLazyTestEnv = require("./helpers/createLazyTestEnv");

View File

@ -1,7 +1,7 @@
"use strict";
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const webpack = require("../");
const rimraf = require("rimraf");
@ -9,12 +9,15 @@ describe("Profiling Plugin", function() {
jest.setTimeout(15000);
it("should handle output path with folder creation", done => {
const finalPath = "test/js/profilingPath/events.json";
const outputPath = path.join(__dirname, "/js/profilingPath");
const outputPath = path.join(__dirname, "js/profilingPath");
const finalPath = path.join(outputPath, "events.json");
rimraf(outputPath, () => {
const compiler = webpack({
context: "/",
context: __dirname,
entry: "./fixtures/a.js",
output: {
path: path.join(__dirname, "js/profilingOut")
},
plugins: [
new webpack.debug.ProfilingPlugin({
outputPath: finalPath

View File

@ -4,7 +4,7 @@
const path = require("path");
const MemoryFs = require("memory-fs");
const webpack = require("..");
const fs = require("fs");
const fs = require("graceful-fs");
const rimraf = require("rimraf");
const createCompiler = config => {

View File

@ -1,6 +1,6 @@
"use strict";
const fs = require("fs");
const fs = require("graceful-fs");
const path = require("path");
const glob = require("glob");
const rootDir = path.resolve(__dirname, "..");

View File

@ -2,7 +2,7 @@
"use strict";
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const webpack = require("..");

View File

@ -2,7 +2,7 @@
"use strict";
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const vm = require("vm");
const mkdirp = require("mkdirp");
const rimraf = require("rimraf");

View File

@ -2,7 +2,7 @@
/*globals describe it */
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const MemoryFs = require("memory-fs");
const webpack = require("..");

View File

@ -2,7 +2,7 @@
"use strict";
const path = require("path");
const fs = require("fs");
const fs = require("graceful-fs");
const vm = require("vm");
const mkdirp = require("mkdirp");
const rimraf = require("rimraf");

View File

@ -1,5 +1,5 @@
"use strict";
const fs = require("fs");
const fs = require("graceful-fs");
const path = require("path");
const check = (expected, actual) => {
@ -26,7 +26,7 @@ const explain = object => {
value = JSON.stringify(value);
}
let msg = `${key} = ${value}`;
if (msg.length > 100) msg = msg.slice(0, 97) + "...";
if (key !== "stack" && msg.length > 100) msg = msg.slice(0, 97) + "...";
return msg;
})
.join("; ");