opensourcemirror
/
webpack
mirror of https://github.com/webpack/webpack.git
1
0
Fork 0
Code Issues Releases Wiki Activity

extract source map before loaders run

This commit is contained in:
Ivan Kopeykin 2022-04-13 11:08:20 +03:00 committed by Nitin Kumar
parent 16c9347f77
commit 4651b265c1
24 changed files with 411 additions and 323 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

299
lib/ExtractSourceMapPlugin.js
View File

@ -1,299 +0,0 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Ivan Kopeykin @vankop
*/
"use strict";
const asyncLib = require("neo-async");
const path = require("path");
const urlUtils = require("url");
const { SourceMapSource } = require("webpack-sources");
const WebpackError = require("./WebpackError");
const ConstDependency = require("./dependencies/ConstDependency");
const { isAbsolute, join } = require("./util/fs");
/** @typedef {import("estree").Comment} CommentNode */
/** @typedef {import("../declarations/WebpackOptions").JavascriptParserOptions} JavascriptParserOptions */
/** @typedef {import("./Compiler")} Compiler */
/** @typedef {import("./javascript/JavascriptParser")} JavascriptParser */
/** @typedef {import("./util/fs").InputFileSystem} InputFileSystem */
// Matches only the last occurrence of sourceMappingURL
const sourceMappingURLRegex = /\s*[#@]\s*sourceMappingURL\s*=\s*([^\s'"]*)\s*/;
/**
* @param {CommentNode[]} comments comments
* @returns {{sourceMappingURL: string, node: CommentNode}|undefined} source mapping
*/
function getSourceMappingURL(comments) {
let match;
let comment;
for (let i = comments.length - 1; i >= 0; i--) {
comment = comments[i];
if (comment.type === "Block") continue;
match = comment.value.match(sourceMappingURLRegex);
if (match) break;
}
if (!match) return;
const sourceMappingURL = match[1] || match[2] || "";
return {
sourceMappingURL: sourceMappingURL
? decodeURI(sourceMappingURL)
: sourceMappingURL,
node: comment
};
}
function getAbsolutePath(fs, context, request, sourceRoot) {
if (sourceRoot) {
if (isAbsolute(sourceRoot)) {
return join(fs, sourceRoot, request);
}
return join(fs, join(fs, context, sourceRoot), request);
}
return join(fs, context, request);
}
function fetchFromDataURL(sourceURL, callback) {
const dataURL = /^data:(?:[^;,]+)?(?:;[^;,]+)*?(?:;(base64))?,(.*)$/i.exec(
sourceURL
);
if (dataURL) {
const encodingName = dataURL[1] ? "base64" : "ascii";
return callback(
null,
Buffer.from(dataURL[2], encodingName).toString("utf-8")
);
}
callback(
new Error(`Failed to parse source map from "data" URL: ${sourceURL}`)
);
}
function fetchFromFilesystem(fs, sourceURL, callback) {
fs.readFile(sourceURL, (err, bufferOrString) => {
if (err)
return callback(
new Error(`Failed to parse source map from '${sourceURL}' file: ${err}`)
);
callback(null, { path: sourceURL, data: bufferOrString.toString("utf-8") });
});
}
function fetchPathsFromFilesystem(
fs,
possibleRequests,
errorsAccumulator = "",
callback
) {
const cb = (error, result) => {
if (error) {
errorsAccumulator += `${error.message}\n\n`;
const tailPossibleRequests = possibleRequests.slice(1);
if (tailPossibleRequests.length === 0) {
error.message = errorsAccumulator;
return callback(error);
}
return fetchPathsFromFilesystem(
fs,
tailPossibleRequests,
errorsAccumulator,
cb
);
}
callback(null, result);
};
fetchFromFilesystem(fs, possibleRequests[0], cb);
}
/**
* @param {InputFileSystem} fs fs
* @param {string} context context
* @param {string} url url
* @param {string|undefined} sourceRoot source root
* @param {function (Error|null, {sourceURL: string, sourceContent: string}=): void} callback callback
* @returns {void}
*/
function fetchFromURL(fs, context, url, sourceRoot, callback) {
// 1. It's an absolute url and it is not `windows` path like `C:\dir\file`
if (/^[a-z][a-z0-9+.-]*:/i.test(url) && !path.win32.isAbsolute(url)) {
const { protocol } = new urlUtils.URL(url);
if (protocol === "data:") {
fetchFromDataURL(url, (err, sourceContent) => {
if (err) return callback(err);
callback(null, { sourceURL: "", sourceContent });
});
} else if (protocol === "file:") {
const sourceURL = urlUtils.fileURLToPath(url);
fetchFromFilesystem(fs, sourceURL, (err, result) => {
if (err) return callback(err);
const { data: sourceContent } = result;
callback(null, { sourceURL, sourceContent });
});
} else {
callback(
new Error(`Failed to parse source map: '${url}' URL is not supported`)
);
}
return;
}
// 2. It's a scheme-relative
if (/^\/\//.test(url)) {
return callback(
new Error(`Failed to parse source map: '${url}' URL is not supported`)
);
}
// 3. Absolute path
if (isAbsolute(url)) {
const possibleRequests = [url];
if (url.startsWith("/") || url.startsWith("\\")) {
possibleRequests.push(
getAbsolutePath(fs, context, url.slice(1), sourceRoot)
);
}
fetchPathsFromFilesystem(fs, possibleRequests, "", (err, result) => {
if (err) return callback(err);
const { path: sourceURL, data: sourceContent } = result;
callback(null, { sourceURL, sourceContent });
});
} else {
// 4. Relative path
const sourceURL = getAbsolutePath(fs, context, url, sourceRoot);
fetchFromFilesystem(fs, sourceURL, (err, result) => {
if (err) return callback(err);
const { data: sourceContent } = result;
callback(null, { sourceURL, sourceContent });
});
}
}
class ExtractSourceMapPlugin {
/**
* @param {Compiler} compiler compiler
*/
apply(compiler) {
compiler.hooks.compilation.tap(
"ExtractSourceMapPlugin",
(compilation, { normalModuleFactory }) => {
const modules = [];
/**
* @param {JavascriptParser} parser parser
* @param {JavascriptParserOptions} parserOptions parser options
*/
const handlerJavascript = (parser, parserOptions) => {
if (parserOptions.extractSourceMap !== true) return;
parser.hooks.finish.tap("ExtractSourceMapPlugin", (_, comments) => {
const mapping = getSourceMappingURL(comments);
if (!mapping || !mapping.sourceMappingURL) return;
const { node, sourceMappingURL } = mapping;
const module = parser.state.module;
const dep = new ConstDependency(
"// extracted source map",
node.range
);
dep.loc = node.loc;
module.addPresentationalDependency(dep);
if (!module.useSourceMap || !("setSourceMap" in module)) return;
const source = module.originalSource();
if (!source) return;
if (source instanceof SourceMapSource) {
const warnings = new WebpackError(
"ExtractSourceMapPlugin: Source already has source map."
);
warnings.details =
"It maybe caused by using 'source-map-loader'.\nRemove it from configuration and try again.";
warnings.loc = node.loc;
module.addWarning(warnings);
return;
}
modules.push([module, sourceMappingURL]);
});
};
compilation.hooks.finishModules.tapAsync(
"ExtractSourceMapPlugin",
(_, callback) => {
if (modules.length === 0) return callback();
asyncLib.each(
modules,
([module, sourceMappingURL], callback) => {
fetchFromURL(
compiler.inputFileSystem,
module.context || "",
sourceMappingURL,
"",
(err, result) => {
if (err) {
module.addWarning(err);
return callback();
}
const { sourceURL, sourceContent } = result;
if (module.buildInfo && module.buildInfo.fileDependencies) {
module.buildInfo.fileDependencies.add(sourceURL);
}
let map;
try {
map = JSON.parse(sourceContent.replace(/^\)\]\}'/, ""));
} catch (parseError) {
module.addWarning(
new WebpackError(
`Failed to parse source map from '${sourceMappingURL}': ${parseError}`
)
);
return callback();
}
// if (map.sections) {
// map = await flattenSourceMap(map);
// }
// add source content?
module.setSourceMap(map, compiler.context);
return callback();
}
);
},
callback
);
}
);
normalModuleFactory.hooks.parser
.for("javascript/auto")
.tap("ExtractSourceMapPlugin", handlerJavascript);
normalModuleFactory.hooks.parser
.for("javascript/esm")
.tap("ExtractSourceMapPlugin", handlerJavascript);
normalModuleFactory.hooks.parser
.for("javascript/dynamic")
.tap("ExtractSourceMapPlugin", handlerJavascript);
}
);
}
}
module.exports = ExtractSourceMapPlugin;

29
lib/NormalModule.js
View File

@ -39,6 +39,7 @@ const {
} = require("./util/comparators");
const createHash = require("./util/createHash");
const { createFakeHook } = require("./util/deprecation");
const extractSourceMap = require("./util/extractSourceMap");
const { join } = require("./util/fs");
const {
contextify,
@ -737,25 +738,6 @@ class NormalModule extends Module {
return new RawSource(content);
}
/**
* @param {SourceMap} sourceMap source map
* @param {string} context build context
*/
setSourceMap(sourceMap, context) {
if (!this._source) throw new Error("There is no source for source map.");
if (!this.useSourceMap) return;
if (this._source instanceof SourceMapSource)
throw new Error("There is source map already.");
let isBuffer = true;
if ("isBuffer" in this._source)
isBuffer = /** @type {RawSource} */ (this._source).isBuffer();
this._source = this.createSource(
context,
isBuffer ? this._source.buffer() : this._source.source(),
sourceMap
);
}
/**
* @param {WebpackOptions} options webpack options
* @param {Compilation} compilation the compilation
@ -856,6 +838,15 @@ class NormalModule extends Module {
if (typeof result !== "string" && !result) {
return callback(new UnhandledSchemeError(scheme, resource));
}
if (this.parserOptions && this.parserOptions.extractSourceMap) {
return extractSourceMap(
result.toString("utf-8"),
loaderContext.fs,
this,
!this.useSourceMap && !this.useSimpleSourceMap,
callback
);
}
return callback(null, result);
});
}

2
lib/WebpackOptionsApply.js
View File

@ -24,7 +24,6 @@ const ConstPlugin = require("./ConstPlugin");
const ExportsInfoApiPlugin = require("./ExportsInfoApiPlugin");
const WebpackIsIncludedPlugin = require("./WebpackIsIncludedPlugin");
const ExtractSourceMapPlugin = require("./ExtractSourceMapPlugin");
const TemplatedPathPlugin = require("./TemplatedPathPlugin");
const UseStrictPlugin = require("./UseStrictPlugin");
const WarnCaseSensitiveModulesPlugin = require("./WarnCaseSensitiveModulesPlugin");
@ -270,7 +269,6 @@ class WebpackOptionsApply extends OptionsApply {
}
}
new ExtractSourceMapPlugin().apply(compiler);
new JavascriptModulesPlugin().apply(compiler);
new JsonModulesPlugin().apply(compiler);
new AssetModulesPlugin().apply(compiler);

252
lib/util/extractSourceMap.js Normal file
View File

@ -0,0 +1,252 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Ivan Kopeykin @vankop
*/
"use strict";
const path = require("path");
const urlUtils = require("url");
const WebpackError = require("../WebpackError");
const { isAbsolute, join } = require("./fs");
/** @typedef {import("../NormalModule")} NormalModule */
/** @typedef {import("./fs").InputFileSystem} InputFileSystem */
// Matches only the last occurrence of sourceMappingURL
const innerRegex = /\s*[#@]\s*sourceMappingURL\s*=\s*([^\s'"]*)\s*/;
/* eslint-disable prefer-template */
const sourceMappingURLRegex = RegExp(
"(?:" +
"/\\*" +
"(?:\\s*\r?\n(?://)?)?" +
"(?:" +
innerRegex.source +
")" +
"\\s*" +
"\\*/" +
"|" +
"//(?:" +
innerRegex.source +
")" +
")" +
"\\s*"
);
/* eslint-enable prefer-template */
/**
* @param {string} code code
* @returns {{replacementString: null|string, sourceMappingURL: null|string}} source mapping url
*/
function getSourceMappingURL(code) {
const lines = code.split(/^/m);
let match;
for (let i = lines.length - 1; i >= 0; i--) {
match = lines[i].match(sourceMappingURLRegex);
if (match) {
break;
}
}
const sourceMappingURL = match ? match[1] || match[2] || "" : null;
return {
sourceMappingURL: sourceMappingURL
? decodeURI(sourceMappingURL)
: sourceMappingURL,
replacementString: match ? match[0] : null
};
}
function getAbsolutePath(fs, context, request, sourceRoot) {
if (sourceRoot) {
if (isAbsolute(sourceRoot)) {
return join(fs, sourceRoot, request);
}
return join(fs, join(fs, context, sourceRoot), request);
}
return join(fs, context, request);
}
function fetchFromDataURL(sourceURL, callback) {
const dataURL = /^data:(?:[^;,]+)?(?:;[^;,]+)*?(?:;(base64))?,(.*)$/i.exec(
sourceURL
);
if (dataURL) {
const encodingName = dataURL[1] ? "base64" : "ascii";
return callback(
null,
Buffer.from(dataURL[2], encodingName).toString("utf-8")
);
}
callback(
new Error(`Failed to parse source map from "data" URL: ${sourceURL}`)
);
}
function fetchFromFilesystem(fs, sourceURL, callback) {
fs.readFile(sourceURL, (err, bufferOrString) => {
if (err)
return callback(
new Error(`Failed to parse source map from '${sourceURL}' file: ${err}`)
);
callback(null, { path: sourceURL, data: bufferOrString.toString("utf-8") });
});
}
function fetchPathsFromFilesystem(
fs,
possibleRequests,
errorsAccumulator = "",
callback
) {
const cb = (error, result) => {
if (error) {
errorsAccumulator += `${error.message}\n\n`;
const tailPossibleRequests = possibleRequests.slice(1);
if (tailPossibleRequests.length === 0) {
error.message = errorsAccumulator;
return callback(error);
}
return fetchPathsFromFilesystem(
fs,
tailPossibleRequests,
errorsAccumulator,
cb
);
}
callback(null, result);
};
fetchFromFilesystem(fs, possibleRequests[0], cb);
}
/**
* @param {InputFileSystem} fs fs
* @param {string} context context
* @param {string} url url
* @param {string|undefined} sourceRoot source root
* @param {function (Error|null, {sourceURL: string, sourceContent: string}=): void} callback callback
* @returns {void}
*/
function fetchFromURL(fs, context, url, sourceRoot, callback) {
// 1. It's an absolute url and it is not `windows` path like `C:\dir\file`
if (/^[a-z][a-z0-9+.-]*:/i.test(url) && !path.win32.isAbsolute(url)) {
const { protocol } = new urlUtils.URL(url);
if (protocol === "data:") {
fetchFromDataURL(url, (err, sourceContent) => {
if (err) return callback(err);
callback(null, { sourceURL: "", sourceContent });
});
} else if (protocol === "file:") {
const sourceURL = urlUtils.fileURLToPath(url);
fetchFromFilesystem(fs, sourceURL, (err, result) => {
if (err) return callback(err);
const { data: sourceContent } = result;
callback(null, { sourceURL, sourceContent });
});
} else {
callback(
new Error(`Failed to parse source map: '${url}' URL is not supported`)
);
}
return;
}
// 2. It's a scheme-relative
if (/^\/\//.test(url)) {
return callback(
new Error(`Failed to parse source map: '${url}' URL is not supported`)
);
}
// 3. Absolute path
if (isAbsolute(url)) {
const possibleRequests = [url];
if (url.startsWith("/") || url.startsWith("\\")) {
possibleRequests.push(
getAbsolutePath(fs, context, url.slice(1), sourceRoot)
);
}
fetchPathsFromFilesystem(fs, possibleRequests, "", (err, result) => {
if (err) return callback(err);
const { path: sourceURL, data: sourceContent } = result;
callback(null, { sourceURL, sourceContent });
});
} else {
// 4. Relative path
const sourceURL = getAbsolutePath(fs, context, url, sourceRoot);
fetchFromFilesystem(fs, sourceURL, (err, result) => {
if (err) return callback(err);
const { data: sourceContent } = result;
callback(null, { sourceURL, sourceContent });
});
}
}
/**
* @param {string} input input data
* @param {InputFileSystem} fs filesystem
* @param {NormalModule} module normal module
* @param {boolean} removeOnly should only remove source map comment
* @param {(err: Error|null, code?: string, sourceMap?: object) => void} callback callback
*/
module.exports = function extractSourceMap(
input,
fs,
module,
removeOnly,
callback
) {
const { sourceMappingURL, replacementString } = getSourceMappingURL(input);
if (!sourceMappingURL) {
callback(null, input);
return;
} else if (removeOnly) {
callback(null, input.replace(replacementString, ""));
return;
}
fetchFromURL(
fs,
module.context || "",
sourceMappingURL,
"",
(err, result) => {
if (err) return callback(err);
const { sourceURL, sourceContent } = result;
if (sourceURL && module.buildInfo && module.buildInfo.fileDependencies) {
module.buildInfo.fileDependencies.add(sourceURL);
}
let map;
try {
map = JSON.parse(sourceContent.replace(/^\)\]\}'/, ""));
} catch (parseError) {
return callback(
new WebpackError(
`Failed to parse source map from '${sourceMappingURL}': ${parseError}`
)
);
}
return callback(null, input.replace(replacementString, ""), map);
}
);
};

2
package.json
View File

@ -21,7 +21,7 @@
"glob-to-regexp": "^0.4.1",
"graceful-fs": "^4.2.9",
"json-parse-even-better-errors": "^2.3.1",
"loader-runner": "^4.2.0",
"loader-runner": "^4.3.0",
"mime-types": "^2.1.27",
"neo-async": "^2.6.2",
"schema-utils": "^3.2.0",

3
test/configCases/source-map/extract-source-map/errors.js Normal file
View File

@ -0,0 +1,3 @@
module.exports = [
/Failed to parse source map/
];

10
test/configCases/source-map/extract-source-map/extract1.js Normal file
View File

@ -0,0 +1,10 @@
const fs = require("fs");
const path = require("path");
require("./test1");
it("should extract source map", () => {
const fileData = fs.readFileSync(path.resolve(__dirname, "bundle1.js.map")).toString("utf-8");
const { sources } = JSON.parse(fileData);
expect(sources).toContain("webpack:///charset-inline-source-map.txt");
});

10
test/configCases/source-map/extract-source-map/extract2.js Normal file
View File

@ -0,0 +1,10 @@
const fs = require("fs");
const path = require("path");
require("./test2");
it("should extract source map", () => {
const fileData = fs.readFileSync(path.resolve(__dirname, "bundle2.js.map")).toString("utf-8");
const { sources } = JSON.parse(fileData);
expect(sources).toContain("webpack:///external-source-map.txt");
});

10
test/configCases/source-map/extract-source-map/extract3.js Normal file
View File

@ -0,0 +1,10 @@
const fs = require("fs");
const path = require("path");
require("./test2");
it("should extract source map", () => {
const fileData = fs.readFileSync(path.resolve(__dirname, "bundle2.js.map")).toString("utf-8");
const { sources } = JSON.parse(fileData);
expect(sources).toContain("webpack:///external-source-map.txt");
});

3
test/configCases/source-map/extract-source-map/infrastructure-log.js Normal file
View File

@ -0,0 +1,3 @@
module.exports = [
/^Pack got invalid because of write to: Compilation\/modules.+no-source-map\.js$/
];

2
test/configCases/source-map/extract-source-map/no-source-map.js Normal file
View File

@ -0,0 +1,2 @@
const a = 1;
//#sourceMappingURL=no-source-map.map

9
test/configCases/source-map/extract-source-map/remove-comment.js Normal file
View File

@ -0,0 +1,9 @@
const fs = require("fs");
require("./test1");
it("should remove sourceMap comment", () => {
expect(
fs.readFileSync(__filename).toString("utf-8")
).not.toMatch(/\/\/\s*@\s*sourceMappingURL/);
});

3
test/configCases/source-map/extract-source-map/test1.js Normal file
View File

@ -0,0 +1,3 @@
const a = 1;
// @ sourceMappingURL = data:application/source-map;charset=utf-8;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY2hhcnNldC1pbmxpbmUtc291cmNlLW1hcC5qcyIsInNvdXJjZXMiOlsiY2hhcnNldC1pbmxpbmUtc291cmNlLW1hcC50eHQiXSwic291cmNlc0NvbnRlbnQiOlsid2l0aCBTb3VyY2VNYXAiXSwibWFwcGluZ3MiOiJBQUFBIn0=
// comment

3
test/configCases/source-map/extract-source-map/test2.js Normal file
View File

@ -0,0 +1,3 @@
const a = 1;
// comment
//#sourceMappingURL=test2.map

1
test/configCases/source-map/extract-source-map/test2.map Normal file
View File

@ -0,0 +1 @@
{"version":3,"file":"test2.js","sources":["external-source-map.txt"],"sourcesContent":["with SourceMap"],"mappings":"AAAA"}

3
test/configCases/source-map/extract-source-map/test3.js Normal file
View File

@ -0,0 +1,3 @@
const a = 1;
// comment
//#sourceMappingURL=/test2.map

58
test/configCases/source-map/extract-source-map/webpack.config.js Normal file
View File

@ -0,0 +1,58 @@
/** @type {import("../../../../").Configuration[]} */
module.exports = [
{
target: "node",
devtool: false,
entry: "./remove-comment",
module: {
rules: [
{
parser: {
extractSourceMap: true
}
}
]
}
},
{
target: "node",
entry: "./extract1",
devtool: "source-map",
module: {
rules: [
{
parser: {
extractSourceMap: true
}
}
]
}
},
{
target: "node",
entry: "./extract2",
devtool: "source-map",
module: {
rules: [
{
parser: {
extractSourceMap: true
}
}
]
}
},
{
entry: "./no-source-map",
devtool: "source-map",
module: {
rules: [
{
parser: {
extractSourceMap: true
}
}
]
}
}
];

3
test/configCases/source-map/extract-source-map2/a.js Normal file
View File

@ -0,0 +1,3 @@
const a = 1;
// comment
//#sourceMappingURL=/a.map

1
test/configCases/source-map/extract-source-map2/a.map Normal file
View File

@ -0,0 +1 @@
{"version":3,"file":"test2.js","sources":["external-source-map.txt"],"sourcesContent":["with SourceMap"],"mappings":"AAAA"}

10
test/configCases/source-map/extract-source-map2/index.js Normal file
View File

@ -0,0 +1,10 @@
const fs = require("fs");
const path = require("path");
require("./a");
it("should extract source map", () => {
const fileData = fs.readFileSync(path.resolve(__dirname, "bundle0.js.map")).toString("utf-8");
const { sources } = JSON.parse(fileData);
expect(sources).toContain("webpack:///external-source-map.txt");
});

3
test/configCases/source-map/extract-source-map2/test.filter.js Normal file
View File

@ -0,0 +1,3 @@
module.exports = function () {
return process.platform !== "win32";
};

15
test/configCases/source-map/extract-source-map2/webpack.config.js Normal file
View File

@ -0,0 +1,15 @@
/** @type {import("../../../../").Configuration} */
module.exports = {
target: "node",
entry: "./index",
devtool: "source-map",
module: {
rules: [
{
parser: {
extractSourceMap: true
}
}
]
}
};

1
types.d.ts vendored
View File

@ -8470,7 +8470,6 @@ declare class NormalModule extends Module {
sourceMap?: any,
associatedObjectForCache?: Object
): Source;
setSourceMap(sourceMap: SourceMap, context: string): void;
markModuleAsErrored(error: WebpackError): void;
applyNoParseRule(rule?: any, content?: any): any;
shouldPreventParsing(noParseRule?: any, request?: any): any;

2
yarn.lock
View File

@ -4361,7 +4361,7 @@ listr2@^5.0.7:
through "^2.3.8"
wrap-ansi "^7.0.0"
loader-runner@^4.2.0:
loader-runner@^4.3.0:
version "4.3.0"
resolved "https://registry.yarnpkg.com/loader-runner/-/loader-runner-4.3.0.tgz#c1b4a163b99f614830353b16755e7149ac2314e1"
integrity sha512-3R/1M+yS3j5ou80Me59j7F9IMs4PXs3VqRrm0TU3AbKPxlmpoY1TNscJV/oGJXo8qCatFGTfDbY6W6ipGOYXfg==