Improve performance of LimitChunkCountPlugin a lot
This commit is contained in:
Tobias Koppers
parent
adfa484921
commit
6f1c685071
|
|
@ -385,7 +385,7 @@ class Chunk {
|
|||
/**
|
||||
*
|
||||
* @param {Chunk} otherChunk the chunk to integrate with
|
||||
* @param {ModuleReason} reason reason why the module is being integrated
|
||||
* @param {string} reason reason why the module is being integrated
|
||||
* @returns {boolean} returns true or false if integration succeeds or fails
|
||||
*/
|
||||
integrate(otherChunk, reason) {
|
||||
|
|
@ -462,6 +462,11 @@ class Chunk {
|
|||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Chunk} a chunk
|
||||
* @param {Chunk} b chunk
|
||||
* @returns {boolean} true, if a is always available when b is reached
|
||||
*/
|
||||
const isAvailable = (a, b) => {
|
||||
const queue = new Set(b.groupsIterable);
|
||||
for (const chunkGroup of queue) {
|
||||
|
|
|
|||
|
|
@ -6,8 +6,33 @@
|
|||
|
||||
const validateOptions = require("schema-utils");
|
||||
const schema = require("../../schemas/plugins/optimize/LimitChunkCountPlugin.json");
|
||||
const LazyBucketSortedSet = require("../util/LazyBucketSortedSet");
|
||||
|
||||
/** @typedef {import("../../declarations/plugins/optimize/LimitChunkCountPlugin").LimitChunkCountPluginOptions} LimitChunkCountPluginOptions */
|
||||
/** @typedef {import("../Chunk")} Chunk */
|
||||
/** @typedef {import("../Compiler")} Compiler */
|
||||
|
||||
/**
|
||||
* @typedef {Object} ChunkCombination
|
||||
* @property {boolean} deleted this is set to true when combination was removed
|
||||
* @property {number} sizeDiff
|
||||
* @property {number} integratedSize
|
||||
* @property {Chunk} a
|
||||
* @property {Chunk} b
|
||||
* @property {number} aIdx
|
||||
* @property {number} bIdx
|
||||
* @property {number} aSize
|
||||
* @property {number} bSize
|
||||
*/
|
||||
|
||||
const addToSetMap = (map, key, value) => {
|
||||
const set = map.get(key);
|
||||
if (set === undefined) {
|
||||
map.set(key, new Set([value]));
|
||||
} else {
|
||||
set.add(value);
|
||||
}
|
||||
};
|
||||
|
||||
class LimitChunkCountPlugin {
|
||||
/**
|
||||
|
|
@ -19,6 +44,11 @@ class LimitChunkCountPlugin {
|
|||
validateOptions(schema, options, "Limit Chunk Count Plugin");
|
||||
this.options = options;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Compiler} compiler the webpack compiler
|
||||
* @returns {void}
|
||||
*/
|
||||
apply(compiler) {
|
||||
const options = this.options;
|
||||
compiler.hooks.compilation.tap("LimitChunkCountPlugin", compilation => {
|
||||
|
|
@ -30,47 +60,169 @@ class LimitChunkCountPlugin {
|
|||
if (maxChunks < 1) return;
|
||||
if (chunks.length <= maxChunks) return;
|
||||
|
||||
let remainingChunksToMerge = chunks.length - maxChunks;
|
||||
|
||||
// order chunks in a deterministic way
|
||||
const orderedChunks = chunks.slice().sort((a, b) => a.compareTo(b));
|
||||
|
||||
const sortedExtendedPairCombinations = orderedChunks
|
||||
.reduce((combinations, a, idx) => {
|
||||
// create combination pairs
|
||||
for (let i = 0; i < idx; i++) {
|
||||
const b = orderedChunks[i];
|
||||
combinations.push([b, a]);
|
||||
}
|
||||
return combinations;
|
||||
}, [])
|
||||
.map(pair => {
|
||||
// extend combination pairs with size and integrated size
|
||||
const a = pair[0].size(options);
|
||||
const b = pair[1].size(options);
|
||||
const ab = pair[0].integratedSize(pair[1], options);
|
||||
return [a + b - ab, ab, pair[0], pair[1], a, b];
|
||||
})
|
||||
.filter(extendedPair => {
|
||||
// create a lazy sorted data structure to keep all combinations
|
||||
// this is large. Size = chunks * (chunks - 1) / 2
|
||||
// It uses a multi layer bucket sort plus normal sort in the last layer
|
||||
// It's also lazy so only accessed buckets are sorted
|
||||
const combinations = new LazyBucketSortedSet(
|
||||
// Layer 1: ordered by largest size benefit
|
||||
c => c.sizeDiff,
|
||||
(a, b) => b - a,
|
||||
// Layer 2: ordered by smallest combined size
|
||||
c => c.integratedSize,
|
||||
(a, b) => a - b,
|
||||
// Layer 3: ordered by position difference in orderedChunk (-> to be deterministic)
|
||||
c => c.bIdx - c.aIdx,
|
||||
(a, b) => a - b,
|
||||
// Layer 4: ordered by position in orderedChunk (-> to be deterministic)
|
||||
(a, b) => a.bIdx - b.bIdx
|
||||
);
|
||||
|
||||
// we keep a mappng from chunk to all combinations
|
||||
// but this mapping is not kept up-to-date with deletions
|
||||
// so `deleted` flag need to be considered when iterating this
|
||||
/** @type {Map<Chunk, Set<ChunkCombination>>} */
|
||||
const combinationsByChunk = new Map();
|
||||
|
||||
orderedChunks.forEach((b, bIdx) => {
|
||||
// create combination pairs with size and integrated size
|
||||
for (let aIdx = 0; aIdx < bIdx; aIdx++) {
|
||||
const a = orderedChunks[aIdx];
|
||||
const integratedSize = a.integratedSize(b, options);
|
||||
|
||||
// filter pairs that do not have an integratedSize
|
||||
// meaning they can NOT be integrated!
|
||||
return extendedPair[1] !== false;
|
||||
})
|
||||
.sort((a, b) => {
|
||||
// sadly javascript does an inplace sort here
|
||||
// sort them by size
|
||||
const diff1 = b[0] - a[0];
|
||||
if (diff1 !== 0) return diff1;
|
||||
const diff2 = a[1] - b[1];
|
||||
if (diff2 !== 0) return diff2;
|
||||
const diff3 = a[2].compareTo(b[2]);
|
||||
if (diff3 !== 0) return diff3;
|
||||
return a[3].compareTo(b[3]);
|
||||
});
|
||||
if (integratedSize === false) continue;
|
||||
|
||||
const pair = sortedExtendedPairCombinations[0];
|
||||
const aSize = a.size(options);
|
||||
const bSize = b.size(options);
|
||||
const c = {
|
||||
deleted: false,
|
||||
sizeDiff: aSize + bSize - integratedSize,
|
||||
integratedSize,
|
||||
a,
|
||||
b,
|
||||
aIdx,
|
||||
bIdx,
|
||||
aSize,
|
||||
bSize
|
||||
};
|
||||
combinations.add(c);
|
||||
addToSetMap(combinationsByChunk, a, c);
|
||||
addToSetMap(combinationsByChunk, b, c);
|
||||
}
|
||||
return combinations;
|
||||
});
|
||||
|
||||
if (pair && pair[2].integrate(pair[3], "limit")) {
|
||||
chunks.splice(chunks.indexOf(pair[3]), 1);
|
||||
return true;
|
||||
// list of modified chunks during this run
|
||||
// combinations affected by this change are skipped to allow
|
||||
// futher optimizations
|
||||
/** @type {Set<Chunk>} */
|
||||
const modifiedChunks = new Set();
|
||||
|
||||
let changed = false;
|
||||
// eslint-disable-next-line no-constant-condition
|
||||
loop: while (true) {
|
||||
const combination = combinations.popFirst();
|
||||
if (combination === undefined) break;
|
||||
|
||||
combination.deleted = true;
|
||||
const { a, b, integratedSize } = combination;
|
||||
|
||||
// skip over pair when
|
||||
// one of the already merged chunks is a parent of one of the chunks
|
||||
if (modifiedChunks.size > 0) {
|
||||
const queue = new Set(a.groupsIterable);
|
||||
for (const group of b.groupsIterable) {
|
||||
queue.add(group);
|
||||
}
|
||||
for (const group of queue) {
|
||||
for (const mChunk of modifiedChunks) {
|
||||
if (mChunk !== a && mChunk !== b && mChunk.isInGroup(group)) {
|
||||
// This is a potential pair which needs recalculation
|
||||
// We can't do that now, but it merge before following pairs
|
||||
// so we leave space for it, and consider chunks as modified
|
||||
// just for the worse case
|
||||
remainingChunksToMerge--;
|
||||
if (remainingChunksToMerge <= 0) break loop;
|
||||
modifiedChunks.add(a);
|
||||
modifiedChunks.add(b);
|
||||
continue loop;
|
||||
}
|
||||
}
|
||||
for (const parent of group.parentsIterable) {
|
||||
queue.add(parent);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// merge the chunks
|
||||
if (a.integrate(b, "limit")) {
|
||||
chunks.splice(chunks.indexOf(b), 1);
|
||||
|
||||
// flag chunk a as modified as further optimization are possible for all children here
|
||||
modifiedChunks.add(a);
|
||||
|
||||
changed = true;
|
||||
remainingChunksToMerge--;
|
||||
if (remainingChunksToMerge <= 0) break;
|
||||
|
||||
// Update all affected combinations
|
||||
// delete all combination with the removed chunk
|
||||
// we will use combinations with the kept chunk instead
|
||||
for (const combination of combinationsByChunk.get(b)) {
|
||||
if (combination.deleted) continue;
|
||||
combination.deleted = true;
|
||||
combinations.delete(combination);
|
||||
}
|
||||
|
||||
// Update combinations with the kept chunk with new sizes
|
||||
for (const combination of combinationsByChunk.get(a)) {
|
||||
if (combination.deleted) continue;
|
||||
if (combination.a === a) {
|
||||
// Update size
|
||||
const newIntegratedSize = a.integratedSize(
|
||||
combination.b,
|
||||
options
|
||||
);
|
||||
if (newIntegratedSize === false) {
|
||||
combination.deleted = true;
|
||||
combinations.delete(combination);
|
||||
continue;
|
||||
}
|
||||
const finishUpdate = combinations.startUpdate(combination);
|
||||
combination.integratedSize = newIntegratedSize;
|
||||
combination.aSize = integratedSize;
|
||||
combination.sizeDiff =
|
||||
combination.bSize + integratedSize - newIntegratedSize;
|
||||
finishUpdate();
|
||||
} else if (combination.b === a) {
|
||||
// Update size
|
||||
const newIntegratedSize = combination.a.integratedSize(
|
||||
a,
|
||||
options
|
||||
);
|
||||
if (newIntegratedSize === false) {
|
||||
combination.deleted = true;
|
||||
combinations.delete(combination);
|
||||
continue;
|
||||
}
|
||||
const finishUpdate = combinations.startUpdate(combination);
|
||||
combination.integratedSize = newIntegratedSize;
|
||||
combination.bSize = integratedSize;
|
||||
combination.sizeDiff =
|
||||
integratedSize + combination.aSize - newIntegratedSize;
|
||||
finishUpdate();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (changed) return true;
|
||||
}
|
||||
);
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,235 @@
|
|||
/*
|
||||
MIT License http://www.opensource.org/licenses/mit-license.php
|
||||
Author Tobias Koppers @sokra
|
||||
*/
|
||||
|
||||
"use strict";
|
||||
|
||||
const SortableSet = require("./SortableSet");
|
||||
|
||||
/**
|
||||
* @template T
|
||||
* @template K
|
||||
* Multi layer bucket sorted set
|
||||
* Supports adding non-existing items (DO NOT ADD ITEM TWICE)
|
||||
* Supports removing exiting items (DO NOT REMOVE ITEM NOT IN SET)
|
||||
* Supports popping the first items according to defined order
|
||||
* Supports iterating all items without order
|
||||
* Supports updating an item in an efficient way
|
||||
* Supports size property, which is the number of items
|
||||
* Items are lazy partially sorted when needed
|
||||
*/
|
||||
class LazyBucketSortedSet {
|
||||
/**
|
||||
* @param {function(T): K} getKey function to get key from item
|
||||
* @param {function(K, K): number} comparator comparator to sort keys
|
||||
* @param {...((function(T): any) | (function(any, any): number))} args more pairs of getKey and comparator plus optional final comparator for the last layer
|
||||
*/
|
||||
constructor(getKey, comparator, ...args) {
|
||||
this._getKey = getKey;
|
||||
this._innerArgs = args;
|
||||
this._leaf = args.length <= 1;
|
||||
this._keys = new SortableSet(undefined, comparator);
|
||||
/** @type {Map<K, LazyBucketSortedSet<T, any> | SortableSet<T>>} */
|
||||
this._map = new Map();
|
||||
this._unsortedItems = new Set();
|
||||
this.size = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {T} item an item
|
||||
* @returns {void}
|
||||
*/
|
||||
add(item) {
|
||||
this.size++;
|
||||
this._unsortedItems.add(item);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {K} key key of item
|
||||
* @param {T} item the item
|
||||
* @returns {void}
|
||||
*/
|
||||
_addInternal(key, item) {
|
||||
let entry = this._map.get(key);
|
||||
if (entry === undefined) {
|
||||
entry = this._leaf
|
||||
? new SortableSet(undefined, this._innerArgs[0])
|
||||
: new /** @type {any} */ (LazyBucketSortedSet)(...this._innerArgs);
|
||||
this._keys.add(key);
|
||||
this._map.set(key, entry);
|
||||
}
|
||||
entry.add(item);
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {T} item an item
|
||||
* @returns {void}
|
||||
*/
|
||||
delete(item) {
|
||||
this.size--;
|
||||
if (this._unsortedItems.has(item)) {
|
||||
this._unsortedItems.delete(item);
|
||||
return;
|
||||
}
|
||||
const key = this._getKey(item);
|
||||
const entry = this._map.get(key);
|
||||
entry.delete(item);
|
||||
if (entry.size === 0) {
|
||||
this._deleteKey(key);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {K} key key to be removed
|
||||
* @returns {void}
|
||||
*/
|
||||
_deleteKey(key) {
|
||||
this._keys.delete(key);
|
||||
this._map.delete(key);
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {T | undefined} an item
|
||||
*/
|
||||
popFirst() {
|
||||
if (this.size === 0) return undefined;
|
||||
this.size--;
|
||||
if (this._unsortedItems.size > 0) {
|
||||
for (const item of this._unsortedItems) {
|
||||
const key = this._getKey(item);
|
||||
this._addInternal(key, item);
|
||||
}
|
||||
this._unsortedItems.clear();
|
||||
}
|
||||
this._keys.sort();
|
||||
const key = this._keys.values().next().value;
|
||||
const entry = this._map.get(key);
|
||||
if (this._leaf) {
|
||||
const leafEntry = /** @type {SortableSet<T>} */ (entry);
|
||||
leafEntry.sort();
|
||||
const item = leafEntry.values().next().value;
|
||||
leafEntry.delete(item);
|
||||
if (leafEntry.size === 0) {
|
||||
this._deleteKey(key);
|
||||
}
|
||||
return item;
|
||||
} else {
|
||||
const nodeEntry = /** @type {LazyBucketSortedSet<T, any>} */ (entry);
|
||||
const item = nodeEntry.popFirst();
|
||||
if (nodeEntry.size === 0) {
|
||||
this._deleteKey(key);
|
||||
}
|
||||
return item;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {T} item to be updated item
|
||||
* @returns {function(true=): void} finish update
|
||||
*/
|
||||
startUpdate(item) {
|
||||
if (this._unsortedItems.has(item)) {
|
||||
return remove => {
|
||||
if (remove) {
|
||||
this._unsortedItems.delete(item);
|
||||
this.size--;
|
||||
return;
|
||||
}
|
||||
};
|
||||
}
|
||||
const key = this._getKey(item);
|
||||
if (this._leaf) {
|
||||
const oldEntry = /** @type {SortableSet<T>} */ (this._map.get(key));
|
||||
return remove => {
|
||||
if (remove) {
|
||||
this.size--;
|
||||
oldEntry.delete(item);
|
||||
if (oldEntry.size === 0) {
|
||||
this._deleteKey(key);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const newKey = this._getKey(item);
|
||||
if (key === newKey) {
|
||||
// This flags the sortable set as unordered
|
||||
oldEntry.add(item);
|
||||
} else {
|
||||
oldEntry.delete(item);
|
||||
if (oldEntry.size === 0) {
|
||||
this._deleteKey(key);
|
||||
}
|
||||
this._addInternal(newKey, item);
|
||||
}
|
||||
};
|
||||
} else {
|
||||
const oldEntry = /** @type {LazyBucketSortedSet<T, any>} */ (this._map.get(
|
||||
key
|
||||
));
|
||||
const finishUpdate = oldEntry.startUpdate(item);
|
||||
return remove => {
|
||||
if (remove) {
|
||||
this.size--;
|
||||
finishUpdate(true);
|
||||
if (oldEntry.size === 0) {
|
||||
this._deleteKey(key);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const newKey = this._getKey(item);
|
||||
if (key === newKey) {
|
||||
finishUpdate();
|
||||
} else {
|
||||
finishUpdate(true);
|
||||
if (oldEntry.size === 0) {
|
||||
this._deleteKey(key);
|
||||
}
|
||||
this._addInternal(newKey, item);
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {Iterator<T>[]} iterators list of iterators to append to
|
||||
* @returns {void}
|
||||
*/
|
||||
_appendIterators(iterators) {
|
||||
if (this._unsortedItems.size > 0)
|
||||
iterators.push(this._unsortedItems[Symbol.iterator]());
|
||||
for (const key of this._keys) {
|
||||
const entry = this._map.get(key);
|
||||
if (this._leaf) {
|
||||
const leafEntry = /** @type {SortableSet<T>} */ (entry);
|
||||
const iterator = leafEntry[Symbol.iterator]();
|
||||
iterators.push(iterator);
|
||||
} else {
|
||||
const nodeEntry = /** @type {LazyBucketSortedSet<T, any>} */ (entry);
|
||||
nodeEntry._appendIterators(iterators);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Iterator<T>} the iterator
|
||||
*/
|
||||
[Symbol.iterator]() {
|
||||
const iterators = [];
|
||||
this._appendIterators(iterators);
|
||||
iterators.reverse();
|
||||
let currentIterator = iterators.pop();
|
||||
return {
|
||||
next: () => {
|
||||
const res = currentIterator.next();
|
||||
if (res.done) {
|
||||
if (iterators.length === 0) return res;
|
||||
currentIterator = iterators.pop();
|
||||
return currentIterator.next();
|
||||
}
|
||||
return res;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = LazyBucketSortedSet;
|
||||
Loading…
Reference in New Issue