opensourcemirror
/
webpack
mirror of https://github.com/webpack/webpack.git
1
0
Fork 0
Code Issues Releases Wiki Activity

refactor: merge generators in asset modules

This commit is contained in:
Sergey Melyukov 2019-11-18 16:29:19 +03:00 committed by Tobias Koppers
parent 2d5844c56e
commit bdc61abf82
24 changed files with 286 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
declarations/plugins/AssetModulesPlugin.d.ts vendored
View File

@ -16,13 +16,20 @@ export type DataUrlFn = (
) => string | Buffer | null;
export interface AssetModulesPluginOptions {
dataUrl?: DataUrlOptions | DataUrlFn;
/**
* The options for data url generator
*/
dataUrl?: false | DataUrlOptions | DataUrlFn;
}
/**
* This interface was referenced by `AssetModulesPluginOptions`'s JSON-Schema
* via the `definition` "DataUrlOptions".
*/
export interface DataUrlOptions {
/**
* Is data url encoding enabled (true by default)
*/
enabled?: boolean;
/**
* Module output encoding
*/

11
examples/asset-advanced/README.md
View File

@ -232,17 +232,16 @@ module.exports = "data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' v
```
Hash: 0a1b2c3d4e5f6a7b8c9d
Version: webpack 5.0.0-beta.7
Asset Size
images/44f5018145f1c78c78e8.svg 656 bytes [emitted] [immutable] [name: (main)]
output.js 5.08 KiB [emitted] [name: main]
Entrypoint main = output.js (images/44f5018145f1c78c78e8.svg)
chunk output.js (main) 1.35 KiB (javascript) 656 bytes (asset) 895 bytes (runtime) [entry] [rendered]
Asset Size
output.js 5.08 KiB [emitted] [name: main]
Entrypoint main = output.js
chunk output.js (main) 1.6 KiB (javascript) 895 bytes (runtime) [entry] [rendered]
> ./example.js main
./example.js 658 bytes [built]
[no exports]
[used exports unknown]
entry ./example.js main
./images/file.svg 656 bytes (asset) 728 bytes (javascript) [built]
./images/file.svg 984 bytes [built]
[used exports unknown]
harmony side effect evaluation ./images/file.svg ./example.js 1:0-36
harmony import specifier ./images/file.svg ./example.js 26:1-4

18
examples/asset-simple/README.md
View File

@ -241,27 +241,25 @@ module.exports = "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5v
```
Hash: 0a1b2c3d4e5f6a7b8c9d
Version: webpack 5.0.0-beta.7
Asset Size
images/1be7b55b29524343503e.svg 656 bytes [emitted] [immutable] [name: (main)]
images/24e804317f239f7906e1.png 14.6 KiB [emitted] [immutable] [name: (main)]
images/afb284cb97b4374bd1fc.jpg 5.89 KiB [emitted] [immutable] [name: (main)]
output.js 14.6 KiB [emitted] [name: main]
Entrypoint main = output.js (images/1be7b55b29524343503e.svg images/24e804317f239f7906e1.png images/afb284cb97b4374bd1fc.jpg)
chunk output.js (main) 9.59 KiB (javascript) 21.1 KiB (asset) 927 bytes (runtime) [entry] [rendered]
Asset Size
images/24e804317f239f7906e1.png 14.6 KiB [emitted] [immutable] [name: (main)]
output.js 14.6 KiB [emitted] [name: main]
Entrypoint main = output.js (images/24e804317f239f7906e1.png)
chunk output.js (main) 10.6 KiB (javascript) 14.6 KiB (asset) 927 bytes (runtime) [entry] [rendered]
> ./example.js main
./example.js 742 bytes [built]
[no exports]
[used exports unknown]
entry ./example.js main
./images/file.jpg 5.89 KiB (asset) 7.89 KiB (javascript) [built]
./images/file.jpg 8.83 KiB [built]
[used exports unknown]
harmony side effect evaluation ./images/file.jpg ./example.js 2:0-36
harmony import specifier ./images/file.jpg ./example.js 28:6-9
./images/file.png 14.6 KiB (asset) 75 bytes (javascript) [built]
./images/file.png 42 bytes (javascript) 14.6 KiB (asset) [built]
[used exports unknown]
harmony side effect evaluation ./images/file.png ./example.js 1:0-36
harmony import specifier ./images/file.png ./example.js 28:1-4
./images/file.svg 656 bytes (asset) 922 bytes (javascript) [built]
./images/file.svg 984 bytes [built]
[used exports unknown]
harmony side effect evaluation ./images/file.svg ./example.js 3:0-36
harmony import specifier ./images/file.svg ./example.js 28:11-14

6
lib/Generator.js
View File

@ -35,9 +35,10 @@ class Generator {
/**
* @abstract
* @param {NormalModule} module fresh module
* @returns {Set<string>} available types (do not mutate)
*/
getTypes() {
getTypes(module) {
throw new AbstractMethodError();
}
@ -73,9 +74,10 @@ class ByTypeGenerator extends Generator {
}
/**
* @param {NormalModule} module fresh module
* @returns {Set<string>} available types (do not mutate)
*/
getTypes() {
getTypes(module) {
return this._types;
}

4
lib/NormalModule.js
View File

@ -772,7 +772,7 @@ class NormalModule extends Module {
* @returns {Set<string>} types availiable (do not mutate)
*/
getSourceTypes() {
return this.generator.getTypes();
return this.generator.getTypes(this);
}
/**
@ -801,7 +801,7 @@ class NormalModule extends Module {
}
const sources = new Map();
for (const type of this.generator.getTypes()) {
for (const type of this.generator.getTypes(this)) {
const source = this.error
? new RawSource(
"throw new Error(" + JSON.stringify(this.error.message) + ");"

87
lib/asset/AssetGenerator.js
View File

@ -5,29 +5,32 @@
"use strict";
const { RawSource } = require("webpack-sources");
const Generator = require("../Generator");
const RuntimeGlobals = require("../RuntimeGlobals");
const encoder = require("./encoder");
/** @typedef {import("webpack-sources").Source} Source */
/** @typedef {import("../../declarations/plugins/AssetModulesPlugin").AssetModulesPluginOptions} AssetModulesPluginOptions */
/** @typedef {import("../Compilation")} Compilation */
/** @typedef {import("../Compiler")} Compiler */
/** @typedef {import("../Generator").GenerateContext} GenerateContext */
/** @typedef {import("../Module")} Module */
/** @typedef {import("../NormalModule")} NormalModule */
/** @typedef {import("../RuntimeTemplate")} RuntimeTemplate */
const TYPES = new Set(["asset"]);
const JS_TYPES = new Set(["javascript"]);
const JS_AND_ASSET_TYPES = new Set(["javascript", "asset"]);
class AssetGenerator extends Generator {
class AssetJavascriptGenerator extends Generator {
/**
* @param {Compilation} compilation the compilation
* @param {AssetModulesPluginOptions} options the options
*/
constructor(options) {
constructor(compilation, options) {
super();
this.options = options;
}
/**
* @returns {Set<string>} available types (do not mutate)
*/
getTypes() {
return TYPES;
this.compilation = compilation;
this.options = encoder.prepareOptions(options);
}
/**
@ -35,8 +38,48 @@ class AssetGenerator extends Generator {
* @param {GenerateContext} generateContext context for generate
* @returns {Source} generated code
*/
generate(module, generateContext) {
return module.originalSource();
generate(module, { chunkGraph, runtimeTemplate, runtimeRequirements, type }) {
if (type === "asset") {
return module.originalSource();
}
runtimeRequirements.add(RuntimeGlobals.module);
if (!encoder.shouldEmitAsset(module, this.options)) {
const encodedSource = encoder.encode(module, this.options);
return new RawSource(
`${RuntimeGlobals.module}.exports = ${JSON.stringify(encodedSource)};`
);
}
const filename = module.nameForCondition();
const { assetModuleFilename } = runtimeTemplate.outputOptions;
const url = this.compilation.getAssetPath(assetModuleFilename, {
module,
filename,
chunkGraph
});
runtimeRequirements.add(RuntimeGlobals.publicPath); // add __webpack_require__.p
// TODO: (hiroppy) use ESM
return new RawSource(
`${RuntimeGlobals.module}.exports = ${
RuntimeGlobals.publicPath
} + ${JSON.stringify(url)};`
);
}
/**
* @param {NormalModule} module fresh module
* @returns {Set<string>} available types (do not mutate)
*/
getTypes(module) {
if (encoder.shouldEmitAsset(module, this.options)) {
return JS_AND_ASSET_TYPES;
}
return JS_TYPES;
}
/**
@ -44,14 +87,26 @@ class AssetGenerator extends Generator {
* @param {string=} type source type
* @returns {number} estimate size of the module
*/
getSize(module, type) {
getSize(module, type = module.type) {
const originalSource = module.originalSource();
if (!originalSource) {
return 0;
}
return originalSource.size();
if (type === "asset") {
return originalSource.size();
}
if (encoder.shouldEmitAsset(module, this.options)) {
// it's only estimated so this number is probably fine
// Example: m.exports=r.p+"0123456789012345678901.ext"
return 42;
} else {
// roughly for data url (a little bit tricky)
return originalSource.size() * 1.5;
}
}
}
module.exports = AssetGenerator;
module.exports = AssetJavascriptGenerator;

89
lib/asset/AssetJavascriptGenerator.js
View File

@ -1,89 +0,0 @@
/*
MIT License http://www.opensource.org/licenses/mit-license.php
Author Yuta Hiroto @hiroppy
*/
"use strict";
const { RawSource } = require("webpack-sources");
const Generator = require("../Generator");
const RuntimeGlobals = require("../RuntimeGlobals");
const encode = require("./encoder");
/** @typedef {import("webpack-sources").Source} Source */
/** @typedef {import("../../declarations/plugins/AssetModulesPlugin").AssetModulesPluginOptions} AssetModulesPluginOptions */
/** @typedef {import("../Compilation")} Compilation */
/** @typedef {import("../Compiler")} Compiler */
/** @typedef {import("../Generator").GenerateContext} GenerateContext */
/** @typedef {import("../NormalModule")} NormalModule */
/** @typedef {import("../RuntimeTemplate")} RuntimeTemplate */
const TYPES = new Set(["javascript"]);
class AssetJavascriptGenerator extends Generator {
/**
* @returns {Set<string>} available types (do not mutate)
*/
getTypes() {
return TYPES;
}
/**
* @param {Compilation} compilation the compilation
* @param {AssetModulesPluginOptions} options the options
*/
constructor(compilation, options) {
super();
this.compilation = compilation;
this.options = options;
}
/**
* @param {NormalModule} module module for which the code should be generated
* @param {GenerateContext} generateContext context for generate
* @returns {Source} generated code
*/
generate(module, { chunkGraph, runtimeTemplate, runtimeRequirements }) {
const encodedSource = encode(module, this.options);
runtimeRequirements.add(RuntimeGlobals.module);
if (encodedSource) {
module.buildMeta.dataUrlAsset = true;
return new RawSource(
`${RuntimeGlobals.module}.exports = ${JSON.stringify(encodedSource)};`
);
}
const filename = module.nameForCondition();
const { assetModuleFilename } = runtimeTemplate.outputOptions;
const url = this.compilation.getAssetPath(assetModuleFilename, {
module,
filename,
chunkGraph
});
runtimeRequirements.add(RuntimeGlobals.publicPath); // add __webpack_require__.p
// TODO: (hiroppy) use ESM
return new RawSource(
`${RuntimeGlobals.module}.exports = ${
RuntimeGlobals.publicPath
} + ${JSON.stringify(url)};`
);
}
/**
* @param {NormalModule} module the module
* @param {string=} type source type
* @returns {number} estimate size of the module
*/
getSize(module, type) {
const source = this.compilation.codeGenerationResults
.get(module)
.sources.get(type);
return source.size();
}
}
module.exports = AssetJavascriptGenerator;

10
lib/asset/AssetModulesPlugin.js
View File

@ -7,10 +7,8 @@
const validateOptions = require("schema-utils");
const schema = require("../../schemas/plugins/AssetModulesPlugin.json");
const Generator = require("../Generator");
const { compareModulesByIdentifier } = require("../util/comparators");
const AssetGenerator = require("./AssetGenerator");
const AssetJavascriptGenerator = require("./AssetJavascriptGenerator");
const AssetParser = require("./AssetParser");
/** @typedef {import("webpack-sources").Source} Source */
@ -41,13 +39,7 @@ class AssetModulesPlugin {
name: "Asset Modules Plugin"
});
return Generator.byType({
asset: new AssetGenerator(generatorOptions),
javascript: new AssetJavascriptGenerator(
compilation,
generatorOptions
)
});
return new AssetGenerator(compilation, generatorOptions);
});
compilation.hooks.renderManifest.tap(plugin, (result, options) => {

87
lib/asset/encoder.js
View File

@ -12,12 +12,73 @@ const path = require("path");
/** @typedef {import("../../declarations/plugins/AssetModulesPlugin").AssetModulesPluginOptions} AssetModulesPluginOptions */
/** @typedef {import("../NormalModule")} NormalModule */
/**
* @type {Map<string|Buffer, string|null>}
*/
const dataUrlFnCache = new Map();
/**
* @param {NormalModule} module the module
* @param {AssetModulesPluginOptions} options the options to encode
* @returns {boolean} should emit additional asset for the module
*/
const shouldEmitAsset = (module, options) => {
const originalSource = module.originalSource();
if (typeof options.dataUrl === "function") {
return (
options.dataUrl.call(null, module, module.nameForCondition()) === false
);
}
if (options.dataUrl === false || options.dataUrl.enabled === false) {
return true;
}
return originalSource.size() > options.dataUrl.maxSize;
};
/**
* @param {AssetModulesPluginOptions} options the options to the encoder
* @returns {AssetModulesPluginOptions} normalized options
*/
const prepareOptions = (options = {}) => {
const dataUrl = options.dataUrl || {};
if (typeof dataUrl === "function") {
return {
dataUrl: (source, resourcePath) => {
if (dataUrlFnCache.has(source)) {
return dataUrlFnCache.get(source);
}
const encoded = dataUrl.call(null, source, resourcePath);
dataUrlFnCache.set(source, encoded);
return encoded;
}
};
}
return {
dataUrl: {
enabled: options.dataUrl !== false && dataUrl.enabled !== false,
encoding: "base64",
maxSize: 8192,
...dataUrl
}
};
};
/**
* @param {NormalModule} module a module to encode
* @param {AssetModulesPluginOptions} options the options to the encoder
* @returns {string|null} encoded source
*/
module.exports = (module, options) => {
const encode = (module, options) => {
if (shouldEmitAsset(module, options)) {
return null;
}
const originalSource = module.originalSource();
let content = originalSource.source();
@ -25,19 +86,11 @@ module.exports = (module, options) => {
return options.dataUrl.call(null, content, module.nameForCondition());
}
const dataUrlOptions = {
encoding: "base64",
maxSize: 8192,
...options.dataUrl
};
if (originalSource.size() > dataUrlOptions.maxSize) {
return null;
}
const encoding = dataUrlOptions.encoding;
const extname = path.extname(module.resource);
const mimeType = dataUrlOptions.mimetype || mime.getType(extname);
// @ts-ignore non-false dataUrl ensures in shouldEmitAsset above
const encoding = options.dataUrl.encoding;
const extname = path.extname(module.nameForCondition());
// @ts-ignore non-false dataUrl ensures in shouldEmitAsset above
const mimeType = options.dataUrl.mimetype || mime.getType(extname);
if (encoding === "base64") {
if (typeof content === "string") {
@ -49,3 +102,9 @@ module.exports = (module, options) => {
return `data:${mimeType}${encoding ? `;${encoding}` : ""},${content}`;
};
module.exports = {
encode,
shouldEmitAsset,
prepareOptions
};

3
lib/javascript/JavascriptGenerator.js
View File

@ -33,9 +33,10 @@ const TYPES = new Set(["javascript"]);
class JavascriptGenerator extends Generator {
/**
* @param {NormalModule} module fresh module
* @returns {Set<string>} available types (do not mutate)
*/
getTypes() {
getTypes(module) {
return TYPES;
}

3
lib/json/JsonGenerator.js
View File

@ -75,9 +75,10 @@ const TYPES = new Set(["javascript"]);
class JsonGenerator extends Generator {
/**
* @param {NormalModule} module fresh module
* @returns {Set<string>} available types (do not mutate)
*/
getTypes() {
getTypes(module) {
return TYPES;
}

4
lib/wasm-async/AsyncWebAssemblyGenerator.js
View File

@ -18,10 +18,12 @@ class AsyncWebAssemblyGenerator extends Generator {
super();
this.options = options;
}
/**
* @param {NormalModule} module fresh module
* @returns {Set<string>} available types (do not mutate)
*/
getTypes() {
getTypes(module) {
return TYPES;
}

4
lib/wasm-async/AsyncWebAssemblyJavascriptGenerator.js
View File

@ -26,10 +26,12 @@ class AsyncWebAssemblyJavascriptGenerator extends Generator {
super();
this.filenameTemplate = filenameTemplate;
}
/**
* @param {NormalModule} module fresh module
* @returns {Set<string>} available types (do not mutate)
*/
getTypes() {
getTypes(module) {
return TYPES;
}

3
lib/wasm/WebAssemblyGenerator.js
View File

@ -389,9 +389,10 @@ class WebAssemblyGenerator extends Generator {
}
/**
* @param {NormalModule} module fresh module
* @returns {Set<string>} available types (do not mutate)
*/
getTypes() {
getTypes(module) {
return TYPES;
}

3
lib/wasm/WebAssemblyJavascriptGenerator.js
View File

@ -24,9 +24,10 @@ const TYPES = new Set(["webassembly"]);
class WebAssemblyJavascriptGenerator extends Generator {
/**
* @param {NormalModule} module fresh module
* @returns {Set<string>} available types (do not mutate)
*/
getTypes() {
getTypes(module) {
return TYPES;
}

8
schemas/plugins/AssetModulesPlugin.json
View File

@ -9,6 +9,10 @@
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": {
"description": "Is data url encoding enabled (true by default)",
"type": "boolean"
},
"encoding": {
"description": "Module output encoding",
"enum": [false, "base64"]
@ -29,7 +33,11 @@
"additionalProperties": false,
"properties": {
"dataUrl": {
"description": "The options for data url generator",
"oneOf": [
{
"enum": [false]
},
{
"$ref": "#/definitions/DataUrlOptions"
},

2
test/Validation.test.js
View File

@ -179,7 +179,7 @@ describe("Validation", () => {
expect(msg).toMatchInlineSnapshot(`
"Invalid configuration object. Webpack has been initialised using a configuration object that does not match the API schema.
- configuration.module.rules[0].oneOf[0] has an unknown property 'passer'. These properties are valid:
object { compiler?, enforce?, exclude?, include?, issuer?, loader?, oneOf?, options?, parser?, realResource?, resolve?, resource?, resourceQuery?, rules?, sideEffects?, test?, type?, use? }
object { compiler?, enforce?, exclude?, generator?, include?, issuer?, loader?, oneOf?, options?, parser?, realResource?, resolve?, resource?, resourceQuery?, rules?, sideEffects?, test?, type?, use? }
-> A rule"
`)
);

16
test/__snapshots__/StatsTestCases.test.js.snap
View File

@ -134,16 +134,14 @@ exports[`StatsTestCases should print correct stats for asset 1`] = `
"Hash: b52cbf090dc450f1a713
Time: Xms
Built at: 1970-04-20 12:42:42
Asset Size
44af8fe384aadccba06e.svg 656 bytes [emitted] [immutable] [name: (main)]
62787d6ac9d673cc8926.png 14.6 KiB [emitted] [immutable] [name: (main)]
bundle.js 3.6 KiB [emitted] [name: main]
c2a9ba2e6ec92fd70245.jpg 5.89 KiB [emitted] [immutable] [name: (main)]
Entrypoint main = bundle.js (44af8fe384aadccba06e.svg 62787d6ac9d673cc8926.png c2a9ba2e6ec92fd70245.jpg)
Asset Size
62787d6ac9d673cc8926.png 14.6 KiB [emitted] [immutable] [name: (main)]
bundle.js 12.2 KiB [emitted] [name: main]
Entrypoint main = bundle.js (62787d6ac9d673cc8926.png)
./index.js 111 bytes [built]
./images/file.png 14.6 KiB (asset) 42 bytes (javascript) [built]
./images/file.svg 656 bytes (asset) 42 bytes (javascript) [built]
./images/file.jpg 5.89 KiB (asset) 42 bytes (javascript) [built]
./images/file.png 68 bytes (javascript) 14.6 KiB (asset) [built]
./images/file.svg 922 bytes [built]
./images/file.jpg 7.89 KiB [built]
+ 3 hidden modules"
`;

5
test/configCases/asset-modules/assetModuleFilename/webpack.config.js
View File

@ -15,7 +15,10 @@ module.exports = {
rules: [
{
test: /\.(png|svg)$/,
type: "asset"
type: "asset",
generator: {
dataUrl: false
}
}
]
},

9
test/configCases/asset-modules/custom-encoder/index.js Normal file
View File

@ -0,0 +1,9 @@
import png from "../_images/file.png";
import svg from "../_images/file.svg";
import jpg from "../_images/file.jpg";
it("should generate various asset types by a custom encoder", () => {
expect(png).toEqual("83d6dab543c538e6621f.png");
expect(jpg).toEqual("00891fd33cfbdc56d145.jpg");
expect(svg).toEqual("data:image/svg+xml;base64,custom-content");
});

26
test/configCases/asset-modules/custom-encoder/webpack.config.js Normal file
View File

@ -0,0 +1,26 @@
module.exports = {
mode: "development",
module: {
rules: [
{
test: /\.(png|jpg)$/,
type: "asset",
generator: {
dataUrl: false
}
},
{
test: /\.svg$/,
type: "asset",
generator: {
dataUrl() {
return "data:image/svg+xml;base64,custom-content";
}
}
}
]
},
experiments: {
asset: true
}
};

9
test/configCases/asset-modules/data-url/index.js Normal file
View File

@ -0,0 +1,9 @@
import png from "../_images/file.png";
import svg from "../_images/file.svg";
import jpg from "../_images/file.jpg";
it("should generate various data-url types", () => {
expect(png).toContain("data:image/png;base64,");
expect(svg).toContain("data:image/svg+xml;base64");
expect(jpg).toContain("data:image/jpeg;base64,");
});

28
test/configCases/asset-modules/data-url/webpack.config.js Normal file
View File

@ -0,0 +1,28 @@
module.exports = {
mode: "development",
module: {
rules: [
{
test: /\.(png|svg)$/,
type: "asset",
generator: {
dataUrl: {
maxSize: Infinity
}
}
},
{
test: /\.jpg$/,
type: "asset",
generator: {
dataUrl: {
maxSize: Infinity
}
}
}
]
},
experiments: {
asset: true
}
};

10
test/configCases/asset-modules/types/webpack.config.js
View File

@ -4,11 +4,17 @@ module.exports = {
rules: [
{
test: /\.(png|svg)$/,
type: "asset"
type: "asset",
generator: {
dataUrl: false
}
},
{
test: /\.jpg$/,
type: "asset"
type: "asset",
generator: {
dataUrl: false
}
}
]
},