opensourcemirror
/
ublock-origin
mirror of https://github.com/gorhill/uBlock.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Rewrite static filtering parser 

This commit is a rewrite of the static filtering parser into a
tree-based data structure, for easier maintenance and better
abstraction of parsed filters.

This simplifies greatly syntax coloring of filters and also
simplify extending filter syntax.

The minimum version of Chromium-based browsers has been raised
to version 73 because of usage of String.matchAll().
This commit is contained in:
Raymond Hill 2023-01-23 16:53:18 -05:00
parent 4564e3a9b8
commit 8ea3b0f64c
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
24 changed files with 2921 additions and 2917 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.jshintrc
View File

@ -2,7 +2,7 @@
"browser": true,
"devel": true,
"eqeqeq": true,
"esversion": 8,
"esversion": 9,
"globals": {
"chrome": false, // global variable in Chromium, Chrome, Opera
"globalThis": false,

20
platform/browser/main.js
View File

@ -29,7 +29,7 @@ import punycode from './lib/punycode.js';
import staticNetFilteringEngine from './js/static-net-filtering.js';
import { FilteringContext } from './js/filtering-context.js';
import { LineIterator } from './js/text-utils.js';
import { StaticFilteringParser } from './js/static-filtering-parser.js';
import * as sfp from './js/static-filtering-parser.js';
import {
CompiledListReader,
@ -40,10 +40,11 @@ import {
function compileList(rawText, writer) {
const lineIter = new LineIterator(rawText);
const parser = new StaticFilteringParser(true);
const compiler = staticNetFilteringEngine.createCompiler(parser);
parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH);
const parser = new sfp.AstFilterParser({
interactive: true,
maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH,
});
const compiler = staticNetFilteringEngine.createCompiler();
while ( lineIter.eot() === false ) {
let line = lineIter.next();
@ -52,13 +53,10 @@ function compileList(rawText, writer) {
if ( lineIter.peek(4) !== ' ' ) { break; }
line = line.slice(0, -2).trim() + lineIter.next().trim();
}
parser.analyze(line);
parser.parse(line);
if ( parser.shouldIgnore() ) { continue; }
if ( parser.category !== parser.CATStaticNetFilter ) { continue; }
if ( parser.patternHasUnicode() && parser.toASCII() === false ) {
continue;
}
if ( parser.isFilter() === false ) { continue; }
if ( parser.isNetworkFilter() === false ) { continue; }
if ( compiler.compile(parser, writer) ) { continue; }
if ( compiler.error !== undefined ) {
console.info(JSON.stringify({

2
platform/chromium/manifest.json
View File

@ -74,7 +74,7 @@
},
"incognito": "split",
"manifest_version": 2,
"minimum_chrome_version": "66.0",
"minimum_chrome_version": "73.0",
"name": "uBlock Origin",
"options_ui": {
"page": "dashboard.html",

4
platform/mv3/make-rulesets.js
View File

@ -30,8 +30,8 @@ import process from 'process';
import { createHash } from 'crypto';
import redirectResourcesMap from './js/redirect-resources.js';
import { dnrRulesetFromRawLists } from './js/static-dnr-filtering.js';
import { StaticFilteringParser } from './js/static-filtering-parser.js';
import { fnameFromFileId } from './js/utils.js';
import * as sfp from './js/static-filtering-parser.js';
/******************************************************************************/
@ -202,7 +202,7 @@ async function fetchAsset(assetDetails) {
);
}
parts = await Promise.all(newParts);
parts = StaticFilteringParser.utils.preparser.expandIncludes(parts, env);
parts = sfp.utils.preparser.expandIncludes(parts, env);
}
const text = parts.join('\n');

19
platform/nodejs/index.js
View File

@ -38,7 +38,7 @@ import publicSuffixList from './lib/publicsuffixlist/publicsuffixlist.js';
import snfe from './js/static-net-filtering.js';
import { FilteringContext } from './js/filtering-context.js';
import { LineIterator } from './js/text-utils.js';
import { StaticFilteringParser } from './js/static-filtering-parser.js';
import * as sfp from './js/static-filtering-parser.js';
import {
CompiledListReader,
@ -117,7 +117,9 @@ function compileList({ name, raw }, compiler, writer, options = {}) {
writer.properties.set('name', name);
}
const { parser } = compiler;
const parser = new sfp.AstFilterParser({
maxTokenLength: snfe.MAX_TOKEN_LENGTH,
});
while ( lineIter.eot() === false ) {
let line = lineIter.next();
@ -125,13 +127,10 @@ function compileList({ name, raw }, compiler, writer, options = {}) {
if ( lineIter.peek(4) !== ' ' ) { break; }
line = line.slice(0, -2).trim() + lineIter.next().trim();
}
parser.analyze(line);
if ( parser.shouldIgnore() ) { continue; }
if ( parser.category !== parser.CATStaticNetFilter ) { continue; }
if ( parser.patternHasUnicode() && parser.toASCII() === false ) {
continue;
}
if ( compiler.compile(writer) ) { continue; }
parser.parse(line);
if ( parser.isFilter() === false ) { continue; }
if ( parser.isNetworkFilter() === false ) { continue; }
if ( compiler.compile(parser, writer) ) { continue; }
if ( compiler.error !== undefined && events !== undefined ) {
options.events.push({
type: 'error',
@ -164,7 +163,7 @@ async function useLists(lists, options = {}) {
if ( typeof compiled !== 'string' || compiled === '' ) {
const writer = new CompiledListWriter();
if ( compiler === null ) {
compiler = snfe.createCompiler(new StaticFilteringParser());
compiler = snfe.createCompiler();
}
compiled = compileList(list, compiler, writer, options);
}

2
platform/opera/manifest.json
View File

@ -73,7 +73,7 @@
},
"incognito": "split",
"manifest_version": 2,
"minimum_opera_version": "53.0",
"minimum_opera_version": "60.0",
"name": "uBlock Origin",
"options_page": "dashboard.html",
"permissions": [

6
src/css/codemirror.css
View File

@ -108,6 +108,12 @@
text-decoration-style: solid;
text-decoration-line: underline;
}
.cm-s-default .cm-unicode {
text-underline-position: under;
text-decoration-color: var(--sf-unicode-ink);
text-decoration-style: dashed;
text-decoration-line: underline;
}
.cm-s-default .cm-tag {
color: var(--sf-tag-ink);
}

1
src/css/themes/default.css
View File

@ -284,6 +284,7 @@
--sf-notice-ink: var(--ink-4);
--sf-readonly-ink: var(--ink-3);
--sf-tag-ink: #006e2e /* h:135 S:100 Luv:40 */;
--sf-unicode-ink: var(--ink-1);
--sf-value-ink: #974900 /* h:30 S:100 Luv:40 */;
--sf-variable-ink: var(--ink-1);
--sf-warning-ink: #e49d00; /* h:50 S:100 Luv:70 */

4
src/js/assets.js
View File

@ -26,8 +26,8 @@
import cacheStorage from './cachestorage.js';
import logger from './logger.js';
import µb from './background.js';
import { StaticFilteringParser } from './static-filtering-parser.js';
import { i18n$ } from './i18n.js';
import * as sfp from './static-filtering-parser.js';
/******************************************************************************/
@ -269,7 +269,7 @@ assets.fetchFilterList = async function(mainlistURL) {
}
if ( result instanceof Object === false ) { continue; }
const content = result.content;
const slices = StaticFilteringParser.utils.preparser.splitter(
const slices = sfp.utils.preparser.splitter(
content,
vAPI.webextFlavor.env
);

4
src/js/background.js
View File

@ -176,8 +176,8 @@ const µBlock = { // jshint ignore:line
// Read-only
systemSettings: {
compiledMagic: 52, // Increase when compiled format changes
selfieMagic: 52, // Increase when selfie format changes
compiledMagic: 54, // Increase when compiled format changes
selfieMagic: 54, // Increase when selfie format changes
},
// https://github.com/uBlockOrigin/uBlock-issues/issues/759#issuecomment-546654501

566
src/js/codemirror/ubo-static-filtering.js
View File

@ -25,7 +25,7 @@
/******************************************************************************/
import { StaticFilteringParser } from '../static-filtering-parser.js';
import * as sfp from '../static-filtering-parser.js';
/******************************************************************************/
@ -39,339 +39,219 @@ let hintHelperRegistered = false;
/******************************************************************************/
CodeMirror.defineMode('ubo-static-filtering', function() {
if ( StaticFilteringParser instanceof Object === false ) { return; }
const parser = new StaticFilteringParser({
if ( sfp.AstFilterParser instanceof Object === false ) { return; }
const astParser = new sfp.AstFilterParser({
interactive: true,
nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
});
const astWalker = astParser.getWalker();
let currentWalkerNode = 0;
let lastNetOptionType = 0;
const reURL = /\bhttps?:\/\/\S+/;
const rePreparseDirectives = /^!#(?:if|endif|include )\b/;
const rePreparseIfDirective = /^(!#if ?)(.*)$/;
let parserSlot = 0;
let netOptionValueMode = false;
const colorCommentSpan = function(stream) {
const { string, pos } = stream;
if ( rePreparseDirectives.test(string) === false ) {
const match = reURL.exec(string.slice(pos));
if ( match !== null ) {
if ( match.index === 0 ) {
stream.pos += match[0].length;
return 'comment link';
}
stream.pos += match.index;
return 'comment';
}
stream.skipToEnd();
return 'comment';
}
const match = rePreparseIfDirective.exec(string);
if ( match === null ) {
stream.skipToEnd();
return 'directive';
}
if ( pos < match[1].length ) {
stream.pos += match[1].length;
return 'directive';
}
stream.skipToEnd();
if ( match[1].endsWith(' ') === false ) {
return 'error strong';
}
if ( preparseDirectiveTokens.size === 0 ) {
return 'positive strong';
}
let token = match[2];
const not = token.startsWith('!');
if ( not ) {
token = token.slice(1);
}
if ( preparseDirectiveTokens.has(token) === false ) {
return 'error strong';
}
if ( not !== preparseDirectiveTokens.get(token) ) {
return 'positive strong';
}
return 'negative strong';
const redirectTokenStyle = node => {
const rawToken = astParser.getNodeString(node);
const { token } = sfp.parseRedirectValue(rawToken);
return redirectNames.has(token) ? 'value' : 'value warning';
};
const colorExtHTMLPatternSpan = function(stream) {
const { i } = parser.patternSpan;
if ( stream.pos === parser.slices[i+1] ) {
stream.pos += 1;
return 'def';
}
stream.skipToEnd();
return 'variable';
};
const colorExtScriptletPatternSpan = function(stream) {
const { pos, string } = stream;
const { i, len } = parser.patternSpan;
const patternBeg = parser.slices[i+1];
if ( pos === patternBeg ) {
stream.pos = pos + 4;
return 'def';
}
if ( len > 3 ) {
if ( pos === patternBeg + 4 ) {
const match = /^[^,)]+/.exec(string.slice(pos));
const token = match && match[0].trim();
if ( token && scriptletNames.has(token) === false ) {
stream.pos = pos + match[0].length;
return 'warning';
}
}
const r = parser.slices[i+len+1] - 1;
if ( pos < r ) {
stream.pos = r;
return 'variable';
}
if ( pos === r ) {
stream.pos = pos + 1;
return 'def';
}
}
stream.skipToEnd();
return 'variable';
};
const colorExtPatternSpan = function(stream) {
if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) {
return colorExtScriptletPatternSpan(stream);
}
if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) {
return colorExtHTMLPatternSpan(stream);
}
stream.skipToEnd();
return 'variable';
};
const colorExtSpan = function(stream) {
if ( parserSlot < parser.optionsAnchorSpan.i ) {
const style = (parser.slices[parserSlot] & parser.BITComma) === 0
? 'value'
: 'def';
stream.pos += parser.slices[parserSlot+2];
parserSlot += 3;
return style;
}
if (
parserSlot >= parser.optionsAnchorSpan.i &&
parserSlot < parser.patternSpan.i
) {
const style = (parser.flavorBits & parser.BITFlavorException) !== 0
? 'tag'
: 'def';
stream.pos += parser.slices[parserSlot+2];
parserSlot += 3;
return `${style} strong`;
}
if (
parserSlot >= parser.patternSpan.i &&
parserSlot < parser.rightSpaceSpan.i
) {
return colorExtPatternSpan(stream);
}
stream.skipToEnd();
return null;
};
const colorNetOptionValueSpan = function(stream, bits) {
const { pos, string } = stream;
let style;
// Warn about unknown redirect tokens.
if (
string.charCodeAt(pos - 1) === 0x3D /* '=' */ &&
/[$,](redirect(-rule)?|rewrite)=$/.test(string.slice(0, pos))
) {
style = 'value';
const end = parser.skipUntil(
parserSlot,
parser.commentSpan.i,
parser.BITComma
);
const raw = parser.strFromSlices(parserSlot, end - 3);
const { token } = StaticFilteringParser.parseRedirectValue(raw);
if ( redirectNames.has(token) === false ) {
style += ' warning';
}
stream.pos += raw.length;
parserSlot = end;
return style;
}
if ( (bits & parser.BITTilde) !== 0 ) {
style = 'keyword strong';
} else if ( (bits & parser.BITPipe) !== 0 ) {
style = 'def';
}
stream.pos += parser.slices[parserSlot+2];
parserSlot += 3;
return style || 'value';
};
// https://github.com/uBlockOrigin/uBlock-issues/issues/760#issuecomment-951146371
// Quick fix: auto-escape commas.
const colorNetOptionSpan = function(stream) {
const [ slotBits, slotPos, slotLen ] =
parser.slices.slice(parserSlot, parserSlot+3);
if ( (slotBits & parser.BITComma) !== 0 ) {
if ( /^,\d*?\}/.test(parser.raw.slice(slotPos)) === false ) {
netOptionValueMode = false;
stream.pos += slotLen;
parserSlot += 3;
return 'def strong';
}
}
if ( netOptionValueMode ) {
return colorNetOptionValueSpan(stream, slotBits);
}
if ( (slotBits & parser.BITTilde) !== 0 ) {
stream.pos += slotLen;
parserSlot += 3;
return 'keyword strong';
}
if ( (slotBits & parser.BITEqual) !== 0 ) {
netOptionValueMode = true;
stream.pos += slotLen;
parserSlot += 3;
return 'def';
}
parserSlot = parser.skipUntil(
parserSlot,
parser.commentSpan.i,
parser.BITComma | parser.BITEqual
);
stream.pos = parser.slices[parserSlot+1];
return 'def';
};
const colorNetSpan = function(stream) {
if ( parserSlot < parser.exceptionSpan.i ) {
stream.pos += parser.slices[parserSlot+2];
parserSlot += 3;
return null;
}
if (
parserSlot === parser.exceptionSpan.i &&
parser.exceptionSpan.len !== 0
) {
stream.pos += parser.slices[parserSlot+2];
parserSlot += 3;
return 'tag strong';
}
if (
parserSlot === parser.patternLeftAnchorSpan.i &&
parser.patternLeftAnchorSpan.len !== 0 ||
parserSlot === parser.patternRightAnchorSpan.i &&
parser.patternRightAnchorSpan.len !== 0
) {
stream.pos += parser.slices[parserSlot+2];
parserSlot += 3;
return 'keyword strong';
}
if (
parserSlot >= parser.patternSpan.i &&
parserSlot < parser.optionsAnchorSpan.i
) {
if ( parser.patternIsRegex() ) {
stream.pos = parser.slices[parser.optionsAnchorSpan.i+1];
parserSlot = parser.optionsAnchorSpan.i;
return parser.patternIsTokenizable()
? 'variable notice'
: 'variable warning';
}
if ( (parser.slices[parserSlot] & (parser.BITAsterisk | parser.BITCaret)) !== 0 ) {
stream.pos += parser.slices[parserSlot+2];
parserSlot += 3;
return 'keyword strong';
}
const nextSlot = parser.skipUntil(
parserSlot + 3,
parser.patternRightAnchorSpan.i,
parser.BITAsterisk | parser.BITCaret
);
stream.pos = parser.slices[nextSlot+1];
parserSlot = nextSlot;
return 'variable';
}
if (
parserSlot === parser.optionsAnchorSpan.i &&
parserSlot < parser.optionsSpan.i !== 0
) {
stream.pos += parser.slices[parserSlot+2];
parserSlot += 3;
return 'def strong';
}
if (
parserSlot >= parser.optionsSpan.i &&
parserSlot < parser.commentSpan.i
) {
return colorNetOptionSpan(stream);
}
if (
parserSlot >= parser.commentSpan.i &&
parser.commentSpan.len !== 0
) {
stream.skipToEnd();
return 'comment';
}
stream.skipToEnd();
return null;
};
const colorSpan = function(stream) {
if ( parser.category === parser.CATNone || parser.shouldIgnore() ) {
stream.skipToEnd();
return 'comment';
}
if ( parser.category === parser.CATComment ) {
return colorCommentSpan(stream);
}
if ( (parser.slices[parserSlot] & parser.BITError) !== 0 ) {
stream.pos += parser.slices[parserSlot+2];
parserSlot += 3;
const colorFromAstNode = function() {
if ( astParser.nodeIsEmptyString(currentWalkerNode) ) { return '+'; }
if ( astParser.getNodeFlags(currentWalkerNode, sfp.NODE_FLAG_ERROR) !== 0 ) {
return 'error';
}
if ( (parser.slices[parserSlot] & parser.BITIgnore) !== 0 ) {
stream.pos += parser.slices[parserSlot+2];
parserSlot += 3;
return 'comment';
}
if ( parser.category === parser.CATStaticExtFilter ) {
const style = colorExtSpan(stream) || '';
let flavor = '';
if ( (parser.flavorBits & parser.BITFlavorExtCosmetic) !== 0 ) {
flavor = 'line-cm-ext-dom';
} else if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) {
flavor = 'line-cm-ext-js';
} else if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) {
flavor = 'line-cm-ext-html';
const nodeType = astParser.getNodeType(currentWalkerNode);
switch ( nodeType ) {
case sfp.NODE_TYPE_WHITESPACE:
return '';
case sfp.NODE_TYPE_COMMENT:
if ( astWalker.canGoDown() ) { break; }
return 'comment';
case sfp.NODE_TYPE_COMMENT_URL:
return 'comment link';
case sfp.NODE_TYPE_IGNORE:
return 'comment';
case sfp.NODE_TYPE_PREPARSE_DIRECTIVE:
case sfp.NODE_TYPE_PREPARSE_DIRECTIVE_VALUE:
return 'directive';
case sfp.NODE_TYPE_PREPARSE_DIRECTIVE_IF_VALUE: {
if ( preparseDirectiveTokens.size === 0 ) {
return 'positive strong';
}
const raw = astParser.getNodeString(currentWalkerNode);
const not = raw.startsWith('!');
const token = not ? raw.slice(1) : raw;
if ( preparseDirectiveTokens.has(token) === false ) {
return 'error strong';
}
return not === preparseDirectiveTokens.get(token)
? 'negative strong'
: 'positive strong';
}
return `${flavor} ${style}`.trim();
case sfp.NODE_TYPE_EXT_OPTIONS_ANCHOR:
return astParser.getFlags(sfp.AST_FLAG_IS_EXCEPTION)
? 'tag strong'
: 'def strong';
case sfp.NODE_TYPE_EXT_DECORATION:
return 'def';
case sfp.NODE_TYPE_EXT_PATTERN_RAW:
if ( astWalker.canGoDown() ) { break; }
return 'variable';
case sfp.NODE_TYPE_EXT_PATTERN_COSMETIC:
case sfp.NODE_TYPE_EXT_PATTERN_HTML:
return 'variable';
case sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER:
case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET:
if ( astWalker.canGoDown() ) { break; }
return 'variable';
case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET_TOKEN: {
const token = astParser.getNodeString(currentWalkerNode);
if ( scriptletNames.has(token) === false ) {
return 'warning';
}
return 'variable';
}
case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARG:
return 'variable';
case sfp.NODE_TYPE_NET_EXCEPTION:
return 'tag strong';
case sfp.NODE_TYPE_NET_PATTERN:
if ( astWalker.canGoDown() ) { break; }
if ( astParser.isRegexPattern() ) {
if ( astParser.getNodeFlags(currentWalkerNode, sfp.NODE_FLAG_PATTERN_UNTOKENIZABLE) !== 0 ) {
return 'variable warning';
}
return 'variable notice';
}
return 'variable';
case sfp.NODE_TYPE_NET_PATTERN_PART:
return 'variable';
case sfp.NODE_TYPE_NET_PATTERN_PART_SPECIAL:
return 'keyword strong';
case sfp.NODE_TYPE_NET_PATTERN_PART_UNICODE:
return 'variable unicode';
case sfp.NODE_TYPE_NET_PATTERN_LEFT_HNANCHOR:
case sfp.NODE_TYPE_NET_PATTERN_LEFT_ANCHOR:
case sfp.NODE_TYPE_NET_PATTERN_RIGHT_ANCHOR:
case sfp.NODE_TYPE_NET_OPTION_NAME_NOT:
return 'keyword strong';
case sfp.NODE_TYPE_NET_OPTIONS_ANCHOR:
case sfp.NODE_TYPE_NET_OPTION_SEPARATOR:
lastNetOptionType = 0;
return 'def strong';
case sfp.NODE_TYPE_NET_OPTION_NAME_UNKNOWN:
lastNetOptionType = 0;
return 'error';
case sfp.NODE_TYPE_NET_OPTION_NAME_1P:
case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT1P:
case sfp.NODE_TYPE_NET_OPTION_NAME_3P:
case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT3P:
case sfp.NODE_TYPE_NET_OPTION_NAME_ALL:
case sfp.NODE_TYPE_NET_OPTION_NAME_BADFILTER:
case sfp.NODE_TYPE_NET_OPTION_NAME_CNAME:
case sfp.NODE_TYPE_NET_OPTION_NAME_CSP:
case sfp.NODE_TYPE_NET_OPTION_NAME_CSS:
case sfp.NODE_TYPE_NET_OPTION_NAME_DENYALLOW:
case sfp.NODE_TYPE_NET_OPTION_NAME_DOC:
case sfp.NODE_TYPE_NET_OPTION_NAME_EHIDE:
case sfp.NODE_TYPE_NET_OPTION_NAME_EMPTY:
case sfp.NODE_TYPE_NET_OPTION_NAME_FONT:
case sfp.NODE_TYPE_NET_OPTION_NAME_FRAME:
case sfp.NODE_TYPE_NET_OPTION_NAME_FROM:
case sfp.NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK:
case sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE:
case sfp.NODE_TYPE_NET_OPTION_NAME_HEADER:
case sfp.NODE_TYPE_NET_OPTION_NAME_IMAGE:
case sfp.NODE_TYPE_NET_OPTION_NAME_IMPORTANT:
case sfp.NODE_TYPE_NET_OPTION_NAME_INLINEFONT:
case sfp.NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT:
case sfp.NODE_TYPE_NET_OPTION_NAME_MATCHCASE:
case sfp.NODE_TYPE_NET_OPTION_NAME_MEDIA:
case sfp.NODE_TYPE_NET_OPTION_NAME_METHOD:
case sfp.NODE_TYPE_NET_OPTION_NAME_MP4:
case sfp.NODE_TYPE_NET_OPTION_NAME_NOOP:
case sfp.NODE_TYPE_NET_OPTION_NAME_OBJECT:
case sfp.NODE_TYPE_NET_OPTION_NAME_OTHER:
case sfp.NODE_TYPE_NET_OPTION_NAME_PING:
case sfp.NODE_TYPE_NET_OPTION_NAME_POPUNDER:
case sfp.NODE_TYPE_NET_OPTION_NAME_POPUP:
case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT:
case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM:
case sfp.NODE_TYPE_NET_OPTION_NAME_SCRIPT:
case sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE:
case sfp.NODE_TYPE_NET_OPTION_NAME_TO:
case sfp.NODE_TYPE_NET_OPTION_NAME_XHR:
case sfp.NODE_TYPE_NET_OPTION_NAME_WEBRTC:
case sfp.NODE_TYPE_NET_OPTION_NAME_WEBSOCKET:
lastNetOptionType = nodeType;
return 'def';
case sfp.NODE_TYPE_NET_OPTION_ASSIGN:
return 'def';
case sfp.NODE_TYPE_NET_OPTION_VALUE:
if ( astWalker.canGoDown() ) { break; }
switch ( lastNetOptionType ) {
case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT:
case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
return redirectTokenStyle(currentWalkerNode);
default:
break;
}
return 'value';
case sfp.NODE_TYPE_OPTION_VALUE_NOT:
return 'keyword strong';
case sfp.NODE_TYPE_OPTION_VALUE_DOMAIN:
return 'value';
case sfp.NODE_TYPE_OPTION_VALUE_SEPARATOR:
return 'def';
default:
break;
}
if ( parser.category === parser.CATStaticNetFilter ) {
const style = colorNetSpan(stream);
return style ? `line-cm-net ${style}` : 'line-cm-net';
}
stream.skipToEnd();
return null;
return '+';
};
return {
lineComment: '!',
token: function(stream) {
let style = '';
if ( stream.sol() ) {
parser.analyze(stream.string);
parser.analyzeExtra();
parserSlot = 0;
netOptionValueMode = false;
astParser.parse(stream.string);
if ( astParser.getFlags(sfp.AST_FLAG_UNSUPPORTED) !== 0 ) {
stream.skipToEnd();
return 'error';
}
if ( astParser.getType() === sfp.AST_TYPE_NONE ) {
stream.skipToEnd();
return 'comment';
}
currentWalkerNode = astWalker.reset();
} else {
currentWalkerNode = astWalker.next();
}
style += colorSpan(stream) || '';
if ( (parser.flavorBits & parser.BITFlavorError) !== 0 ) {
style += ' line-background-error';
let style = '';
while ( currentWalkerNode !== 0 ) {
style = colorFromAstNode(stream);
if ( style !== '+' ) { break; }
currentWalkerNode = astWalker.next();
}
if ( style === '+' ) {
stream.skipToEnd();
return null;
}
stream.pos = astParser.getNodeStringEnd(currentWalkerNode);
if ( astParser.isNetworkFilter() ) {
return style ? `line-cm-net ${style}` : 'line-cm-net';
}
if ( astParser.isExtendedFilter() ) {
let flavor = '';
if ( astParser.isCosmeticFilter() ) {
flavor = 'line-cm-ext-dom';
} else if ( astParser.isScriptletFilter() ) {
flavor = 'line-cm-ext-js';
} else if ( astParser.isHtmlFilter() ) {
flavor = 'line-cm-ext-html';
}
if ( flavor !== '' ) {
style = `${flavor} ${style}`;
}
}
style = style.trim();
return style !== '' ? style : null;
@ -409,9 +289,7 @@ CodeMirror.defineMode('ubo-static-filtering', function() {
initHints();
}
},
get parser() {
return parser;
},
parser: astParser,
};
});
@ -421,11 +299,14 @@ CodeMirror.defineMode('ubo-static-filtering', function() {
// https://codemirror.net/demo/complete.html
const initHints = function() {
if ( StaticFilteringParser instanceof Object === false ) { return; }
if ( sfp.AstFilterParser instanceof Object === false ) { return; }
const parser = new StaticFilteringParser();
const astParser = new sfp.AstFilterParser({
interactive: true,
nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
});
const proceduralOperatorNames = new Map(
Array.from(parser.proceduralOperatorTokens)
Array.from(sfp.proceduralOperatorTokens)
.filter(item => (item[1] & 0b01) !== 0)
);
const excludedHints = new Set([
@ -560,16 +441,16 @@ const initHints = function() {
}
const assignPos = seedRight.indexOf('=');
if ( assignPos !== -1 ) { seedRight = seedRight.slice(0, assignPos); }
const isException = parser.isException();
const isException = astParser.isException();
const hints = [];
for ( let [ text, bits ] of parser.netOptionTokenDescriptors ) {
for ( let [ text, desc ] of sfp.netOptionTokenDescriptors ) {
if ( excludedHints.has(text) ) { continue; }
if ( isNegated && (bits & parser.OPTCanNegate) === 0 ) { continue; }
if ( isNegated && desc.canNegate !== true ) { continue; }
if ( isException ) {
if ( (bits & parser.OPTBlockOnly) !== 0 ) { continue; }
if ( desc.blockOnly ) { continue; }
} else {
if ( (bits & parser.OPTAllowOnly) !== 0 ) { continue; }
if ( (assignPos === -1) && (bits & parser.OPTMustAssign) !== 0 ) {
if ( desc.allowOnly ) { continue; }
if ( (assignPos === -1) && desc.mustAssign ) {
text += '=';
}
}
@ -588,8 +469,11 @@ const initHints = function() {
};
const getNetHints = function(cursor, line) {
const patternNode = astParser.getBranchFromType(sfp.NODE_TYPE_NET_PATTERN_RAW);
if ( patternNode === 0 ) { return; }
const patternEnd = astParser.getNodeStringEnd(patternNode);
const beg = cursor.ch;
if ( beg <= parser.slices[parser.optionsAnchorSpan.i+1] ) {
if ( beg <= patternEnd ) {
return getNetPatternHints(cursor, line);
}
const lineBefore = line.slice(0, beg);
@ -650,7 +534,7 @@ const initHints = function() {
const matchRight = /^([^)]*)/.exec(line.slice(beg));
if ( matchLeft === null || matchRight === null ) { return; }
const hints = [];
for ( const hint of parser.removableHTTPHeaders ) {
for ( const hint of sfp.removableHTTPHeaders ) {
hints.push(hint);
}
return pickBestHints(cursor, matchLeft[1], matchRight[1], hints);
@ -697,29 +581,29 @@ const initHints = function() {
CodeMirror.registerHelper('hint', 'ubo-static-filtering', function(cm) {
const cursor = cm.getCursor();
const line = cm.getLine(cursor.line);
parser.analyze(line);
if ( parser.category === parser.CATStaticExtFilter ) {
astParser.parse(line);
if ( astParser.isExtendedFilter() ) {
const anchorNode = astParser.getBranchFromType(sfp.NODE_TYPE_EXT_OPTIONS_ANCHOR);
if ( anchorNode === 0 ) { return; }
let hints;
if ( cursor.ch <= parser.slices[parser.optionsAnchorSpan.i+1] ) {
if ( cursor.ch <= astParser.getNodeStringBeg(anchorNode) ) {
hints = getOriginHints(cursor, line);
} else if ( parser.hasFlavor(parser.BITFlavorExtScriptlet) ) {
} else if ( astParser.isScriptletFilter() ) {
hints = getExtScriptletHints(cursor, line);
} else if ( parser.hasFlavor(parser.BITFlavorExtResponseHeader) ) {
} else if ( astParser.isResponseheaderFilter() ) {
hints = getExtHeaderHints(cursor, line);
} else {
hints = getExtSelectorHints(cursor, line);
}
return hints;
}
if ( parser.category === parser.CATStaticNetFilter ) {
if ( astParser.isNetworkFilter() ) {
return getNetHints(cursor, line);
}
if ( parser.category === parser.CATComment ) {
if ( astParser.isComment() ) {
return getCommentHints(cursor, line);
}
if ( parser.category === parser.CATNone ) {
return getOriginHints(cursor, line);
}
return getOriginHints(cursor, line);
});
};

2
src/js/cosmetic-filtering.js
View File

@ -322,7 +322,7 @@ FilterContainer.prototype.compile = function(parser, writer) {
// Negated hostname means the filter applies to all non-negated hostnames
// of same filter OR globally if there is no non-negated hostnames.
let applyGlobally = true;
for ( const { hn, not, bad } of parser.extOptions() ) {
for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
if ( bad ) { continue; }
if ( not === false ) {
applyGlobally = false;

17
src/js/epicker-ui.js
View File

@ -26,7 +26,7 @@
import './codemirror/ubo-static-filtering.js';
import { hostnameFromURI } from './uri-utils.js';
import { StaticFilteringParser } from './static-filtering-parser.js';
import * as sfp from './static-filtering-parser.js';
/******************************************************************************/
/******************************************************************************/
@ -110,13 +110,12 @@ const rawFilterFromTextarea = function() {
const filterFromTextarea = function() {
const filter = rawFilterFromTextarea();
if ( filter === '' ) { return ''; }
const sfp = staticFilteringParser;
sfp.analyze(filter);
sfp.analyzeExtra();
if ( sfp.shouldDiscard() ) { return '!'; }
if ( sfp.category === sfp.CATStaticExtFilter ) {
if ( sfp.hasFlavor(sfp.BITFlavorExtCosmetic) === false ) { return '!'; }
} else if ( sfp.category !== sfp.CATStaticNetFilter ) {
const parser = staticFilteringParser;
parser.parse(filter);
if ( parser.isFilter() === false ) { return '!'; }
if ( parser.isExtendedFilter() ) {
if ( parser.isCosmeticFilter() === false ) { return '!'; }
} else if ( parser.isNetworkFilter() === false ) {
return '!';
}
return filter;
@ -829,7 +828,7 @@ const startPicker = function() {
$id('candidateFilters').addEventListener('click', onCandidateClicked);
$stor('#resultsetDepth input').addEventListener('input', onDepthChanged);
$stor('#resultsetSpecificity input').addEventListener('input', onSpecificityChanged);
staticFilteringParser = new StaticFilteringParser({
staticFilteringParser = new sfp.AstFilterParser({
interactive: true,
nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
});

14
src/js/html-filtering.js
View File

@ -26,8 +26,8 @@
import logger from './logger.js';
import µb from './background.js';
import { sessionFirewall } from './filtering-engines.js';
import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js';
import * as sfp from './static-filtering-parser.js';
/******************************************************************************/
@ -314,7 +314,11 @@ htmlFilteringEngine.freeze = function() {
};
htmlFilteringEngine.compile = function(parser, writer) {
const { raw, compiled, exception } = parser.result;
const isException = parser.isException();
const root = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_HTML);
const headerName = parser.getNodeString(root);
const { raw, compiled } = parser.result;
if ( compiled === undefined ) {
const who = writer.properties.get('name') || '?';
logger.writeOne({
@ -329,7 +333,7 @@ htmlFilteringEngine.compile = function(parser, writer) {
// Only exception filters are allowed to be global.
if ( parser.hasOptions() === false ) {
if ( exception ) {
if ( isException ) {
writer.push([ 64, '', 1, compiled ]);
}
return;
@ -337,10 +341,10 @@ htmlFilteringEngine.compile = function(parser, writer) {
// TODO: Mind negated hostnames, they are currently discarded.
for ( const { hn, not, bad } of parser.extOptions() ) {
for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
if ( bad ) { continue; }
let kind = 0;
if ( exception ) {
if ( isException ) {
if ( not ) { continue; }
kind |= 0b01;
}

19
src/js/httpheader-filtering.js
View File

@ -27,8 +27,8 @@ import logger from './logger.js';
import µb from './background.js';
import { entityFromDomain } from './uri-utils.js';
import { sessionFirewall } from './filtering-engines.js';
import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js';
import * as sfp from './static-filtering-parser.js';
/******************************************************************************/
@ -88,16 +88,17 @@ httpheaderFilteringEngine.freeze = function() {
httpheaderFilteringEngine.compile = function(parser, writer) {
writer.select('HTTPHEADER_FILTERS');
const { compiled, exception } = parser.result;
const headerName = compiled.slice(15, -1);
const isException = parser.isException();
const root = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_RESPONSEHEADER);
const headerName = parser.getNodeString(root);
// Tokenless is meaningful only for exception filters.
if ( headerName === '' && exception === false ) { return; }
if ( headerName === '' && isException === false ) { return; }
// Only exception filters are allowed to be global.
if ( parser.hasOptions() === false ) {
if ( exception ) {
writer.push([ 64, '', 1, compiled ]);
if ( isException ) {
writer.push([ 64, '', 1, headerName ]);
}
return;
}
@ -106,16 +107,16 @@ httpheaderFilteringEngine.compile = function(parser, writer) {
// Ignore instances of exception filter with negated hostnames,
// because there is no way to create an exception to an exception.
for ( const { hn, not, bad } of parser.extOptions() ) {
for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
if ( bad ) { continue; }
let kind = 0;
if ( exception ) {
if ( isException ) {
if ( not ) { continue; }
kind |= 1;
} else if ( not ) {
kind |= 1;
}
writer.push([ 64, hn, kind, compiled ]);
writer.push([ 64, hn, kind, headerName ]);
}
};

6
src/js/messaging.js
View File

@ -43,7 +43,7 @@ import { denseBase64 } from './base64-custom.js';
import { dnrRulesetFromRawLists } from './static-dnr-filtering.js';
import { i18n$ } from './i18n.js';
import { redirectEngine } from './redirect-engine.js';
import { StaticFilteringParser } from './static-filtering-parser.js';
import * as sfp from './static-filtering-parser.js';
import {
permanentFirewall,
@ -1515,9 +1515,9 @@ const onMessage = function(request, sender, callback) {
if ( (request.hintUpdateToken || 0) === 0 ) {
response.redirectResources = redirectEngine.getResourceDetails();
response.preparseDirectiveTokens =
StaticFilteringParser.utils.preparser.getTokens(vAPI.webextFlavor.env);
sfp.utils.preparser.getTokens(vAPI.webextFlavor.env);
response.preparseDirectiveHints =
StaticFilteringParser.utils.preparser.getHints();
sfp.utils.preparser.getHints();
response.expertMode = µb.hiddenSettings.filterAuthorMode;
}
if ( request.hintUpdateToken !== µb.pageStoresToken ) {

17
src/js/redirect-resources.js
View File

@ -117,13 +117,6 @@ export default new Map([
[ 'monkeybroker.js', {
alias: 'd3pkae9owd2lcf.cloudfront.net/mb105.js',
} ],
[ 'noeval.js', {
data: 'text',
} ],
[ 'noeval-silent.js', {
alias: 'silent-noeval.js',
data: 'text',
} ],
[ 'nobab.js', {
alias: 'bab-defuser.js',
data: 'text',
@ -131,6 +124,13 @@ export default new Map([
[ 'nobab2.js', {
data: 'text',
} ],
[ 'noeval.js', {
data: 'text',
} ],
[ 'noeval-silent.js', {
alias: 'silent-noeval.js',
data: 'text',
} ],
[ 'nofab.js', {
alias: 'fuckadblock.js-3.2.0',
data: 'text',
@ -145,6 +145,9 @@ export default new Map([
alias: 'noopmp4-1s',
data: 'blob',
} ],
[ 'noop.css', {
data: 'text',
} ],
[ 'noop.html', {
alias: 'noopframe',
} ],

12
src/js/reverselookup.js
View File

@ -26,8 +26,8 @@
import staticNetFilteringEngine from './static-net-filtering.js';
import µb from './background.js';
import { CompiledListWriter } from './static-filtering-io.js';
import { StaticFilteringParser } from './static-filtering-parser.js';
import { i18n$ } from './i18n.js';
import * as sfp from './static-filtering-parser.js';
import {
domainFromHostname,
@ -134,14 +134,14 @@ const fromNetFilter = async function(rawFilter) {
if ( typeof rawFilter !== 'string' || rawFilter === '' ) { return; }
const writer = new CompiledListWriter();
const parser = new StaticFilteringParser({
const parser = new sfp.AstFilterParser({
nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH,
});
parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH);
parser.analyze(rawFilter);
parser.parse(rawFilter);
const compiler = staticNetFilteringEngine.createCompiler(parser);
if ( compiler.compile(writer) === false ) { return; }
const compiler = staticNetFilteringEngine.createCompiler();
if ( compiler.compile(parser, writer) === false ) { return; }
await initWorker();

53
src/js/scriptlet-filtering.js
View File

@ -27,8 +27,8 @@ import logger from './logger.js';
import µb from './background.js';
import { redirectEngine } from './redirect-engine.js';
import { sessionFirewall } from './filtering-engines.js';
import { StaticExtFilteringHostnameDB } from './static-ext-filtering-db.js';
import * as sfp from './static-filtering-parser.js';
import {
domainFromHostname,
@ -117,25 +117,28 @@ const contentscriptCode = (( ) => {
// TODO: Probably should move this into StaticFilteringParser
// https://github.com/uBlockOrigin/uBlock-issues/issues/1031
// Normalize scriptlet name to its canonical, unaliased name.
const normalizeRawFilter = function(rawFilter) {
const rawToken = rawFilter.slice(4, -1);
const rawEnd = rawToken.length;
let end = rawToken.indexOf(',');
if ( end === -1 ) { end = rawEnd; }
const token = rawToken.slice(0, end).trim();
const alias = token.endsWith('.js') ? token.slice(0, -3) : token;
let normalized = redirectEngine.aliases.get(`${alias}.js`);
normalized = normalized === undefined
? alias
: normalized.slice(0, -3);
let beg = end + 1;
while ( beg < rawEnd ) {
end = rawToken.indexOf(',', beg);
if ( end === -1 ) { end = rawEnd; }
normalized += ', ' + rawToken.slice(beg, end).trim();
beg = end + 1;
const normalizeRawFilter = function(parser) {
const root = parser.getBranchFromType(sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET);
const walker = parser.getWalker(root);
const args = [];
for ( let node = walker.next(); node !== 0; node = walker.next() ) {
switch ( parser.getNodeType(node) ) {
case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET_TOKEN:
case sfp.NODE_TYPE_EXT_PATTERN_SCRIPTLET_ARG:
args.push(parser.getNodeString(node));
break;
default:
break;
}
}
return `+js(${normalized})`;
walker.dispose();
if ( args.length !== 0 ) {
const full = `${args[0]}.js`;
if ( redirectEngine.aliases.has(full) ) {
args[0] = redirectEngine.aliases.get(full).slice(0, -3);
}
}
return `+js(${args.join(', ')})`;
};
const lookupScriptlet = function(rawToken, reng, toInject) {
@ -228,14 +231,14 @@ scriptletFilteringEngine.compile = function(parser, writer) {
writer.select('SCRIPTLET_FILTERS');
// Only exception filters are allowed to be global.
const { raw, exception } = parser.result;
const normalized = normalizeRawFilter(raw);
const isException = parser.isException();
const normalized = normalizeRawFilter(parser);
// Tokenless is meaningful only for exception filters.
if ( normalized === '+js()' && exception === false ) { return; }
if ( normalized === '+js()' && isException === false ) { return; }
if ( parser.hasOptions() === false ) {
if ( exception ) {
if ( isException ) {
writer.push([ 32, '', 1, normalized ]);
}
return;
@ -245,10 +248,10 @@ scriptletFilteringEngine.compile = function(parser, writer) {
// Ignore instances of exception filter with negated hostnames,
// because there is no way to create an exception to an exception.
for ( const { hn, not, bad } of parser.extOptions() ) {
for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
if ( bad ) { continue; }
let kind = 0;
if ( exception ) {
if ( isException ) {
if ( not ) { continue; }
kind |= 1;
} else if ( not ) {

42
src/js/static-dnr-filtering.js
View File

@ -25,7 +25,7 @@
import staticNetFilteringEngine from './static-net-filtering.js';
import { LineIterator } from './text-utils.js';
import { StaticFilteringParser } from './static-filtering-parser.js';
import * as sfp from './static-filtering-parser.js';
import {
CompiledListReader,
@ -87,19 +87,17 @@ const keyFromSelector = selector => {
/******************************************************************************/
function addExtendedToDNR(context, parser) {
if ( parser.category !== parser.CATStaticExtFilter ) { return false; }
if ( parser.isExtendedFilter() === false ) { return false; }
// Scriptlet injection
if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) {
if ( (parser.flavorBits & parser.BITFlavorUnsupported) !== 0 ) {
return;
}
if ( parser.isScriptletFilter() ) {
if ( parser.hasOptions() === false ) { return; }
if ( context.scriptletFilters === undefined ) {
context.scriptletFilters = new Map();
}
const { raw, exception } = parser.result;
for ( const { hn, not, bad } of parser.extOptions() ) {
const exception = parser.isException();
const raw = parser.getTypeString(sfp.NODE_TYPE_EXT_PATTERN_RAW);
for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
if ( bad ) { continue; }
if ( exception ) { continue; }
let details = context.scriptletFilters.get(raw);
@ -166,7 +164,7 @@ function addExtendedToDNR(context, parser) {
if ( context.specificCosmeticFilters === undefined ) {
context.specificCosmeticFilters = new Map();
}
for ( const { hn, not, bad } of parser.extOptions() ) {
for ( const { hn, not, bad } of parser.getExtFilterDomainIterator() ) {
if ( bad ) { continue; }
let { compiled, exception, raw } = parser.result;
if ( exception ) { continue; }
@ -209,15 +207,13 @@ function addToDNR(context, list) {
const env = context.env || [];
const writer = new CompiledListWriter();
const lineIter = new LineIterator(
StaticFilteringParser.utils.preparser.prune(list.text, env)
sfp.utils.preparser.prune(list.text, env)
);
const parser = new StaticFilteringParser({
const parser = new sfp.AstFilterParser({
nativeCssHas: env.includes('native_css_has'),
badTypes: [ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE ],
});
const compiler = staticNetFilteringEngine.createCompiler(parser);
// Can't enforce `redirect-rule=` with DNR
compiler.excludeOptions([ parser.OPTTokenRedirectRule ]);
const compiler = staticNetFilteringEngine.createCompiler();
writer.properties.set('name', list.name);
compiler.start(writer);
@ -229,22 +225,18 @@ function addToDNR(context, list) {
line = line.slice(0, -2).trim() + lineIter.next().trim();
}
parser.analyze(line);
parser.parse(line);
if ( parser.shouldIgnore() ) { continue; }
if ( parser.isFilter() === false ) { continue; }
if ( parser.hasError() ) { continue; }
if ( parser.category !== parser.CATStaticNetFilter ) {
if ( parser.isExtendedFilter() ) {
addExtendedToDNR(context, parser);
continue;
}
if ( parser.isNetworkFilter() === false ) { continue; }
// https://github.com/gorhill/uBlock/issues/2599
// convert hostname to punycode if needed
if ( parser.patternHasUnicode() && parser.toASCII() === false ) {
continue;
}
if ( compiler.compile(writer) ) { continue; }
if ( compiler.compile(parser, writer) ) { continue; }
if ( compiler.error !== undefined ) {
context.invalid.add(compiler.error);

24
src/js/static-ext-filtering.js
View File

@ -95,26 +95,25 @@ staticExtFilteringEngine.freeze = function() {
};
staticExtFilteringEngine.compile = function(parser, writer) {
if ( parser.category !== parser.CATStaticExtFilter ) { return false; }
if ( parser.isExtendedFilter() === false ) { return false; }
if ( (parser.flavorBits & parser.BITFlavorUnsupported) !== 0 ) {
const who = writer.properties.get('name') || '?';
if ( parser.hasError() ) {
logger.writeOne({
realm: 'message',
type: 'error',
text: `Invalid extended filter in ${who}: ${parser.raw}`
text: `Invalid extended filter in ${writer.properties.get('name') || '?'}: ${parser.raw}`
});
return true;
}
// Scriptlet injection
if ( (parser.flavorBits & parser.BITFlavorExtScriptlet) !== 0 ) {
if ( parser.isScriptletFilter() ) {
scriptletFilteringEngine.compile(parser, writer);
return true;
}
// Response header filtering
if ( (parser.flavorBits & parser.BITFlavorExtResponseHeader) !== 0 ) {
if ( parser.isResponseheaderFilter() ) {
httpheaderFilteringEngine.compile(parser, writer);
return true;
}
@ -122,13 +121,22 @@ staticExtFilteringEngine.compile = function(parser, writer) {
// HTML filtering
// TODO: evaluate converting Adguard's `$$` syntax into uBO's HTML
// filtering syntax.
if ( (parser.flavorBits & parser.BITFlavorExtHTML) !== 0 ) {
if ( parser.isHtmlFilter() ) {
htmlFilteringEngine.compile(parser, writer);
return true;
}
// Cosmetic filtering
cosmeticFilteringEngine.compile(parser, writer);
if ( parser.isCosmeticFilter() ) {
cosmeticFilteringEngine.compile(parser, writer);
return true;
}
logger.writeOne({
realm: 'message',
type: 'error',
text: `Unknown extended filter in ${writer.properties.get('name') || '?'}: ${parser.raw}`
});
return true;
};

4476
src/js/static-filtering-parser.js
View File

File diff suppressed because it is too large Load Diff

498
src/js/static-net-filtering.js
View File

@ -29,8 +29,8 @@ import { queueTask, dropTask } from './tasks.js';
import BidiTrieContainer from './biditrie.js';
import HNTrieContainer from './hntrie.js';
import { sparseBase64 } from './base64-custom.js';
import { StaticFilteringParser } from './static-filtering-parser.js';
import { CompiledListReader } from './static-filtering-io.js';
import * as sfp from './static-filtering-parser.js';
import {
domainFromHostname,
@ -178,6 +178,24 @@ const typeValueToDNRTypeName = [
'other',
];
const MODIFIER_TYPE_REDIRECT = 1;
const MODIFIER_TYPE_REDIRECTRULE = 2;
const MODIFIER_TYPE_REMOVEPARAM = 3;
const MODIFIER_TYPE_CSP = 4;
const modifierTypeFromName = new Map([
[ 'redirect', MODIFIER_TYPE_REDIRECT ],
[ 'redirect-rule', MODIFIER_TYPE_REDIRECTRULE ],
[ 'removeparam', MODIFIER_TYPE_REMOVEPARAM ],
[ 'csp', MODIFIER_TYPE_CSP ],
]);
const modifierNameFromType = new Map([
[ MODIFIER_TYPE_REDIRECT, 'redirect' ],
[ MODIFIER_TYPE_REDIRECTRULE, 'redirect-rule' ],
[ MODIFIER_TYPE_REMOVEPARAM, 'removeparam' ],
[ MODIFIER_TYPE_CSP, 'csp' ],
]);
//const typeValueFromCatBits = catBits => (catBits >>> TypeBitsOffset) & 0b11111;
@ -1244,7 +1262,7 @@ class FilterRegex {
if ( rule.condition === undefined ) {
rule.condition = {};
}
if ( StaticFilteringParser.utils.regex.isRE2(args[1]) === false ) {
if ( sfp.utils.regex.isRE2(args[1]) === false ) {
dnrAddRuleError(rule, `regexFilter is not RE2-compatible: ${args[1]}`);
}
rule.condition.regexFilter = args[1];
@ -2001,7 +2019,7 @@ class FilterModifier {
static dnrFromCompiled(args, rule) {
rule.__modifierAction = args[1];
rule.__modifierType = StaticFilteringParser.netOptionTokenNames.get(args[2]);
rule.__modifierType = modifierNameFromType.get(args[2]);
rule.__modifierValue = args[3];
}
@ -2010,7 +2028,7 @@ class FilterModifier {
}
static logData(idata, details) {
let opt = StaticFilteringParser.netOptionTokenNames.get(filterData[idata+2]);
let opt = modifierNameFromType.get(filterData[idata+2]);
const refs = filterRefs[filterData[idata+3]];
if ( refs.value !== '' ) {
opt += `=${refs.value}`;
@ -2019,7 +2037,7 @@ class FilterModifier {
}
static dumpInfo(idata) {
const s = StaticFilteringParser.netOptionTokenNames.get(filterData[idata+2]);
const s = modifierNameFromType.get(filterData[idata+2]);
const refs = filterRefs[filterData[idata+3]];
if ( refs.value === '' ) { return s; }
return `${s}=${refs.value}`;
@ -2797,7 +2815,7 @@ class FilterOnHeaders {
static match(idata) {
const refs = filterRefs[filterData[idata+1]];
if ( refs.$parsed === null ) {
refs.$parsed = StaticFilteringParser.parseHeaderValue(refs.headerOpt);
refs.$parsed = sfp.parseHeaderValue(refs.headerOpt);
}
const { bad, name, not, re, value } = refs.$parsed;
if ( bad ) { return false; }
@ -3017,39 +3035,42 @@ const urlTokenizer = new (class {
/******************************************************************************/
class FilterCompiler {
constructor(parser, other = undefined) {
this.parser = parser;
constructor(other = undefined) {
if ( other !== undefined ) {
return Object.assign(this, other);
}
this.reBadCSP = /(?:=|;)\s*report-(?:to|uri)\b/;
this.reToken = /[%0-9A-Za-z]+/g;
this.fromDomainOptList = [];
this.toDomainOptList = [];
this.tokenIdToNormalizedType = new Map([
[ parser.OPTTokenCname, bitFromType('cname') ],
[ parser.OPTTokenCss, bitFromType('stylesheet') ],
[ parser.OPTTokenDoc, bitFromType('main_frame') ],
[ parser.OPTTokenFont, bitFromType('font') ],
[ parser.OPTTokenFrame, bitFromType('sub_frame') ],
[ parser.OPTTokenGenericblock, bitFromType('unsupported') ],
[ parser.OPTTokenGhide, bitFromType('generichide') ],
[ parser.OPTTokenImage, bitFromType('image') ],
[ parser.OPTTokenInlineFont, bitFromType('inline-font') ],
[ parser.OPTTokenInlineScript, bitFromType('inline-script') ],
[ parser.OPTTokenMedia, bitFromType('media') ],
[ parser.OPTTokenObject, bitFromType('object') ],
[ parser.OPTTokenOther, bitFromType('other') ],
[ parser.OPTTokenPing, bitFromType('ping') ],
[ parser.OPTTokenPopunder, bitFromType('popunder') ],
[ parser.OPTTokenPopup, bitFromType('popup') ],
[ parser.OPTTokenScript, bitFromType('script') ],
[ parser.OPTTokenShide, bitFromType('specifichide') ],
[ parser.OPTTokenXhr, bitFromType('xmlhttprequest') ],
[ parser.OPTTokenWebrtc, bitFromType('unsupported') ],
[ parser.OPTTokenWebsocket, bitFromType('websocket') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_CNAME, bitFromType('cname') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_CSS, bitFromType('stylesheet') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_DOC, bitFromType('main_frame') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_FONT, bitFromType('font') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_FRAME, bitFromType('sub_frame') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK, bitFromType('unsupported') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE, bitFromType('generichide') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_IMAGE, bitFromType('image') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_INLINEFONT, bitFromType('inline-font') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT, bitFromType('inline-script') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_MEDIA, bitFromType('media') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_OBJECT, bitFromType('object') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_OTHER, bitFromType('other') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_PING, bitFromType('ping') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_POPUNDER, bitFromType('popunder') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_POPUP, bitFromType('popup') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_SCRIPT, bitFromType('script') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE, bitFromType('specifichide') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_XHR, bitFromType('xmlhttprequest') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_WEBRTC, bitFromType('unsupported') ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_WEBSOCKET, bitFromType('websocket') ],
]);
this.modifierIdToNormalizedId = new Map([
[ sfp.NODE_TYPE_NET_OPTION_NAME_CSP, MODIFIER_TYPE_CSP ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT, MODIFIER_TYPE_REDIRECT ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE, MODIFIER_TYPE_REDIRECTRULE ],
[ sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM, MODIFIER_TYPE_REMOVEPARAM ],
]);
this.excludedOptionSet = new Set();
// These top 100 "bad tokens" are collated using the "miss" histogram
// from tokenHistograms(). The "score" is their occurrence among the
// 200K+ URLs used in the benchmark and executed against default
@ -3182,6 +3203,7 @@ class FilterCompiler {
this.denyallowOpt = '';
this.headerOpt = undefined;
this.isPureHostname = false;
this.isGeneric = false;
this.isRegex = false;
this.strictParty = 0;
this.token = '*';
@ -3203,7 +3225,7 @@ class FilterCompiler {
}
clone() {
return new FilterCompiler(this.parser, this);
return new FilterCompiler(this);
}
normalizeRegexSource(s) {
@ -3215,12 +3237,6 @@ class FilterCompiler {
return '';
}
excludeOptions(options) {
for ( const option of options ) {
this.excludedOptionSet.add(option);
}
}
processMethodOption(value) {
for ( const method of value.split('|') ) {
if ( method.charCodeAt(0) === 0x7E /* '~' */ ) {
@ -3264,21 +3280,12 @@ class FilterCompiler {
this.party |= firstParty ? FirstParty : ThirdParty;
}
processHostnameList(s, modeBits, out = []) {
let beg = 0;
let slen = s.length;
processHostnameList(iter, out = []) {
let i = 0;
while ( beg < slen ) {
let end = s.indexOf('|', beg);
if ( end === -1 ) { end = slen; }
const hn = this.parser.normalizeHostnameValue(
s.slice(beg, end),
modeBits
);
if ( hn !== undefined ) {
out[i] = hn; i += 1;
}
beg = end + 1;
for ( const { hn, not, bad } of iter ) {
if ( bad ) { return ''; }
out[i] = not ? `~${hn}` : hn;
i += 1;
}
out.length = i;
return i === 1 ? out[0] : out.join('|');
@ -3286,146 +3293,206 @@ class FilterCompiler {
processModifierOption(modifier, value) {
if ( this.modifyType !== undefined ) { return false; }
this.modifyType = modifier;
const normalized = this.modifierIdToNormalizedId.get(modifier);
if ( normalized === undefined ) { return false; }
this.modifyType = normalized;
this.modifyValue = value || '';
return true;
}
processOptions() {
const { parser } = this;
for ( let { id, val, not } of parser.netOptions() ) {
switch ( id ) {
case parser.OPTToken1p:
this.processPartyOption(true, not);
processCspOption(value) {
this.modifyType = MODIFIER_TYPE_CSP;
this.modifyValue = value || '';
this.optionUnitBits |= this.CSP_BIT;
return true;
}
processOptionWithValue(parser, id) {
switch ( id ) {
case sfp.NODE_TYPE_NET_OPTION_NAME_CSP:
if ( this.processCspOption(parser.getNetOptionValue(id)) === false ) { return false; }
break;
case parser.OPTToken1pStrict:
this.strictParty = this.strictParty === -1 ? 0 : 1;
this.optionUnitBits |= this.STRICT_PARTY_BIT;
case sfp.NODE_TYPE_NET_OPTION_NAME_DENYALLOW:
this.denyallowOpt = this.processHostnameList(
parser.getNetFilterDenyallowOptionIterator(),
);
if ( this.denyallowOpt === '' ) { return false; }
this.optionUnitBits |= this.DENYALLOW_BIT;
break;
case parser.OPTToken3p:
this.processPartyOption(false, not);
break;
case parser.OPTToken3pStrict:
this.strictParty = this.strictParty === 1 ? 0 : -1;
this.optionUnitBits |= this.STRICT_PARTY_BIT;
break;
case parser.OPTTokenAll:
this.processTypeOption(-1);
break;
// https://github.com/uBlockOrigin/uAssets/issues/192
case parser.OPTTokenBadfilter:
this.badFilter = true;
break;
case parser.OPTTokenCsp:
if ( this.processModifierOption(id, val) === false ) {
return false;
}
if ( val !== undefined && this.reBadCSP.test(val) ) {
return false;
}
this.optionUnitBits |= this.CSP_BIT;
break;
// https://github.com/gorhill/uBlock/issues/2294
// Detect and discard filter if domain option contains
// nonsensical characters.
case parser.OPTTokenFrom:
case sfp.NODE_TYPE_NET_OPTION_NAME_FROM:
this.fromDomainOpt = this.processHostnameList(
val,
0b1010,
parser.getNetFilterFromOptionIterator(),
this.fromDomainOptList
);
if ( this.fromDomainOpt === '' ) { return false; }
this.optionUnitBits |= this.FROM_BIT;
break;
case parser.OPTTokenTo:
case sfp.NODE_TYPE_NET_OPTION_NAME_HEADER: {
this.headerOpt = parser.getNetOptionValue(id) || '';
this.optionUnitBits |= this.HEADER_BIT;
break;
}
case sfp.NODE_TYPE_NET_OPTION_NAME_METHOD:
this.processMethodOption(parser.getNetOptionValue(id));
this.optionUnitBits |= this.METHOD_BIT;
break;
case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT:
if ( this.action === AllowAction ) {
id = sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE;
}
if ( this.processModifierOption(id, parser.getNetOptionValue(id)) === false ) {
return false;
}
this.optionUnitBits |= this.REDIRECT_BIT;
break;
case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
if ( this.processModifierOption(id, parser.getNetOptionValue(id)) === false ) {
return false;
}
this.optionUnitBits |= this.REDIRECT_BIT;
break;
case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM:
if ( this.processModifierOption(id, parser.getNetOptionValue(id)) === false ) {
return false;
}
this.optionUnitBits |= this.REMOVEPARAM_BIT;
break;
case sfp.NODE_TYPE_NET_OPTION_NAME_TO:
this.toDomainOpt = this.processHostnameList(
val,
0b1010,
parser.getNetFilterToOptionIterator(),
this.toDomainOptList
);
if ( this.toDomainOpt === '' ) { return false; }
this.optionUnitBits |= this.TO_BIT;
break;
case parser.OPTTokenDenyAllow:
this.denyallowOpt = this.processHostnameList(val, 0b0000);
if ( this.denyallowOpt === '' ) { return false; }
this.optionUnitBits |= this.DENYALLOW_BIT;
default:
break;
// https://www.reddit.com/r/uBlockOrigin/comments/d6vxzj/
// Add support for `elemhide`. Rarely used but it happens.
case parser.OPTTokenEhide:
this.processTypeOption(parser.OPTTokenShide, not);
this.processTypeOption(parser.OPTTokenGhide, not);
}
return true;
}
process(parser) {
// important!
this.reset();
if ( parser.hasError() ) {
return this.FILTER_INVALID;
}
if ( parser.isException() ) {
this.action = AllowAction;
}
if ( parser.isLeftHnAnchored() ) {
this.anchor |= 0b100;
} else if ( parser.isLeftAnchored() ) {
this.anchor |= 0b010;
}
if ( parser.isRightAnchored() ) {
this.anchor |= 0b001;
}
this.pattern = parser.getNetPattern();
if ( parser.isHostnamePattern() ) {
this.isPureHostname = true;
} else if ( parser.isGenericPattern() ) {
this.isGeneric = true;
} else if ( parser.isRegexPattern() ) {
this.isRegex = true;
}
for ( const type of parser.getNodeTypes() ) {
switch ( type ) {
case sfp.NODE_TYPE_NET_OPTION_NAME_1P:
this.processPartyOption(true, parser.isNegatedOption(type));
break;
case parser.OPTTokenHeader:
this.headerOpt = val !== undefined ? val : '';
this.optionUnitBits |= this.HEADER_BIT;
case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT1P:
this.strictParty = this.strictParty === -1 ? 0 : 1;
this.optionUnitBits |= this.STRICT_PARTY_BIT;
break;
case parser.OPTTokenImportant:
if ( this.action === AllowAction ) { return false; }
case sfp.NODE_TYPE_NET_OPTION_NAME_3P:
this.processPartyOption(false, parser.isNegatedOption(type));
break;
case sfp.NODE_TYPE_NET_OPTION_NAME_STRICT3P:
this.strictParty = this.strictParty === 1 ? 0 : -1;
this.optionUnitBits |= this.STRICT_PARTY_BIT;
break;
case sfp.NODE_TYPE_NET_OPTION_NAME_ALL:
this.processTypeOption(-1);
break;
case sfp.NODE_TYPE_NET_OPTION_NAME_BADFILTER:
this.badFilter = true;
break;
case sfp.NODE_TYPE_NET_OPTION_NAME_CNAME:
case sfp.NODE_TYPE_NET_OPTION_NAME_CSS:
case sfp.NODE_TYPE_NET_OPTION_NAME_DOC:
case sfp.NODE_TYPE_NET_OPTION_NAME_FONT:
case sfp.NODE_TYPE_NET_OPTION_NAME_FRAME:
case sfp.NODE_TYPE_NET_OPTION_NAME_GENERICBLOCK:
case sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE:
case sfp.NODE_TYPE_NET_OPTION_NAME_IMAGE:
case sfp.NODE_TYPE_NET_OPTION_NAME_INLINEFONT:
case sfp.NODE_TYPE_NET_OPTION_NAME_INLINESCRIPT:
case sfp.NODE_TYPE_NET_OPTION_NAME_MEDIA:
case sfp.NODE_TYPE_NET_OPTION_NAME_OBJECT:
case sfp.NODE_TYPE_NET_OPTION_NAME_OTHER:
case sfp.NODE_TYPE_NET_OPTION_NAME_PING:
case sfp.NODE_TYPE_NET_OPTION_NAME_POPUNDER:
case sfp.NODE_TYPE_NET_OPTION_NAME_POPUP:
case sfp.NODE_TYPE_NET_OPTION_NAME_SCRIPT:
case sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE:
case sfp.NODE_TYPE_NET_OPTION_NAME_XHR:
case sfp.NODE_TYPE_NET_OPTION_NAME_WEBRTC:
case sfp.NODE_TYPE_NET_OPTION_NAME_WEBSOCKET:
this.processTypeOption(type, parser.isNegatedOption(type));
break;
case sfp.NODE_TYPE_NET_OPTION_NAME_CSP:
case sfp.NODE_TYPE_NET_OPTION_NAME_DENYALLOW:
case sfp.NODE_TYPE_NET_OPTION_NAME_FROM:
case sfp.NODE_TYPE_NET_OPTION_NAME_HEADER:
case sfp.NODE_TYPE_NET_OPTION_NAME_METHOD:
case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT:
case sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE:
case sfp.NODE_TYPE_NET_OPTION_NAME_REMOVEPARAM:
case sfp.NODE_TYPE_NET_OPTION_NAME_TO:
if ( this.processOptionWithValue(parser, type) === false ) {
return this.FILTER_INVALID;
}
break;
case sfp.NODE_TYPE_NET_OPTION_NAME_EHIDE: {
const not = parser.isNegatedOption(type);
this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_SHIDE, not);
this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_GHIDE, not);
break;
}
case sfp.NODE_TYPE_NET_OPTION_NAME_EMPTY: {
const id = this.action === AllowAction
? sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE
: sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT;
if ( this.processModifierOption(id, 'empty') === false ) {
return this.FILTER_INVALID;
}
this.optionUnitBits |= this.REDIRECT_BIT;
break;
}
case sfp.NODE_TYPE_NET_OPTION_NAME_IMPORTANT:
this.optionUnitBits |= this.IMPORTANT_BIT;
this.action = BlockImportant;
break;
// Used by Adguard:
// https://kb.adguard.com/en/general/how-to-create-your-own-ad-filters#empty-modifier
case parser.OPTTokenEmpty:
id = this.action === AllowAction
? parser.OPTTokenRedirectRule
: parser.OPTTokenRedirect;
if ( this.processModifierOption(id, 'empty') === false ) {
return false;
}
this.optionUnitBits |= this.REDIRECT_BIT;
break;
case parser.OPTTokenMatchCase:
case sfp.NODE_TYPE_NET_OPTION_NAME_MATCHCASE:
this.patternMatchCase = true;
break;
case parser.OPTTokenMp4:
id = this.action === AllowAction
? parser.OPTTokenRedirectRule
: parser.OPTTokenRedirect;
case sfp.NODE_TYPE_NET_OPTION_NAME_MP4: {
const id = this.action === AllowAction
? sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECTRULE
: sfp.NODE_TYPE_NET_OPTION_NAME_REDIRECT;
if ( this.processModifierOption(id, 'noopmp4-1s') === false ) {
return false;
return this.FILTER_INVALID;
}
this.optionUnitBits |= this.REDIRECT_BIT;
break;
case parser.OPTTokenNoop:
break;
case parser.OPTTokenRemoveparam:
if ( this.processModifierOption(id, val) === false ) {
return false;
}
this.optionUnitBits |= this.REMOVEPARAM_BIT;
break;
case parser.OPTTokenRedirect:
if ( this.action === AllowAction ) {
id = parser.OPTTokenRedirectRule;
}
if ( this.processModifierOption(id, val) === false ) {
return false;
}
this.optionUnitBits |= this.REDIRECT_BIT;
break;
case parser.OPTTokenRedirectRule:
if ( this.excludedOptionSet.has(parser.OPTTokenRedirectRule) ) {
return false;
}
if ( this.processModifierOption(id, val) === false ) {
return false;
}
this.optionUnitBits |= this.REDIRECT_BIT;
break;
case parser.OPTTokenMethod:
this.processMethodOption(val);
this.optionUnitBits |= this.METHOD_BIT;
break;
case parser.OPTTokenInvalid:
return false;
}
default:
if ( this.tokenIdToNormalizedType.has(id) === false ) {
return false;
}
this.processTypeOption(id, not);
break;
}
}
@ -3452,10 +3519,10 @@ class FilterCompiler {
}
// CSP directives implicitly apply only to document/subdocument.
if ( this.modifyType === this.parser.OPTTokenCsp ) {
if ( this.modifyType === MODIFIER_TYPE_CSP ) {
if ( this.typeBits === 0 ) {
this.processTypeOption(this.parser.OPTTokenDoc, false);
this.processTypeOption(this.parser.OPTTokenFrame, false);
this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_DOC, false);
this.processTypeOption(sfp.NODE_TYPE_NET_OPTION_NAME_FRAME, false);
}
}
@ -3464,85 +3531,29 @@ class FilterCompiler {
// toggle off `unsupported` bit.
if ( this.typeBits & unsupportedTypeBit ) {
this.typeBits &= ~unsupportedTypeBit;
if ( this.typeBits === 0 ) { return false; }
if ( this.typeBits === 0 ) { return this.FILTER_UNSUPPORTED; }
}
return true;
}
process() {
// important!
this.reset();
if ( this.parser.hasError() ) {
return this.FILTER_INVALID;
}
// Filters which pattern is a single character other than `*` and have
// no narrowing options are discarded as invalid.
if ( this.parser.patternIsDubious() ) {
return this.FILTER_INVALID;
}
// block or allow filter?
// Important: this must be executed before parsing options
if ( this.parser.isException() ) {
this.action = AllowAction;
}
this.isPureHostname = this.parser.patternIsPlainHostname();
// Plain hostname? (from HOSTS file)
if ( this.isPureHostname && this.parser.hasOptions() === false ) {
this.pattern = this.parser.patternToLowercase();
if ( this.isPureHostname && parser.hasOptions() === false ) {
this.anchor |= 0b100;
return this.FILTER_OK;
}
// options
if ( this.parser.hasOptions() && this.processOptions() === false ) {
return this.FILTER_UNSUPPORTED;
}
// regex?
if ( this.parser.patternIsRegex() ) {
this.isRegex = true;
// https://github.com/gorhill/uBlock/issues/1246
// If the filter is valid, use the corrected version of the
// source string -- this ensure reverse-lookup will work fine.
this.pattern = this.normalizeRegexSource(this.parser.getNetPattern());
if ( this.pattern === '' ) {
return this.FILTER_UNSUPPORTED;
}
if ( this.isRegex ) {
return this.FILTER_OK;
}
const pattern = this.parser.patternIsMatchAll()
? '*'
: this.parser.patternToLowercase();
if ( this.parser.patternIsLeftHostnameAnchored() ) {
this.anchor |= 0b100;
} else if ( this.parser.patternIsLeftAnchored() ) {
this.anchor |= 0b010;
}
if ( this.parser.patternIsRightAnchored() ) {
this.anchor |= 0b001;
if ( this.isGeneric ) {
this.wildcardPos = this.pattern.indexOf('*');
this.caretPos = this.pattern.indexOf('^');
}
if ( this.parser.patternHasWildcard() ) {
this.wildcardPos = pattern.indexOf('*');
}
if ( this.parser.patternHasCaret() ) {
this.caretPos = pattern.indexOf('^');
}
if ( pattern.length > 1024 ) {
if ( this.pattern.length > 1024 ) {
return this.FILTER_UNSUPPORTED;
}
this.pattern = pattern;
return this.FILTER_OK;
}
@ -3556,9 +3567,7 @@ class FilterCompiler {
makeToken() {
if ( this.pattern === '*' ) {
if ( this.modifyType !== this.parser.OPTTokenRemoveparam ) {
return;
}
if ( this.modifyType !== MODIFIER_TYPE_REMOVEPARAM ) { return; }
return this.extractTokenFromQuerypruneValue();
}
if ( this.isRegex ) {
@ -3607,7 +3616,7 @@ class FilterCompiler {
// Mind `\b` directives: `/\bads\b/` should result in token being `ads`,
// not `bads`.
extractTokenFromRegex(pattern) {
pattern = StaticFilteringParser.utils.regex.toTokenizableStr(pattern);
pattern = sfp.utils.regex.toTokenizableStr(pattern);
this.reToken.lastIndex = 0;
let bestToken;
let bestBadness = 0x7FFFFFFF;
@ -3682,8 +3691,8 @@ class FilterCompiler {
s.charCodeAt(l-2) === 0x2E /* '.' */;
}
compile(writer) {
const r = this.process();
compile(parser, writer) {
const r = this.process(parser);
// Ignore non-static network filters
if ( r === this.FILTER_INVALID ) { return false; }
@ -3691,7 +3700,7 @@ class FilterCompiler {
// Ignore filters with unsupported options
if ( r === this.FILTER_UNSUPPORTED ) {
const who = writer.properties.get('name') || '?';
this.error = `Invalid network filter in ${who}: ${this.parser.raw}`;
this.error = `Invalid network filter in ${who}: ${parser.raw}`;
return false;
}
@ -3704,8 +3713,8 @@ class FilterCompiler {
// Reminder:
// `redirect=` is a combination of a `redirect-rule` filter and a
// block filter.
if ( this.modifyType === this.parser.OPTTokenRedirect ) {
this.modifyType = this.parser.OPTTokenRedirectRule;
if ( this.modifyType === MODIFIER_TYPE_REDIRECT ) {
this.modifyType = MODIFIER_TYPE_REDIRECTRULE;
const parsedBlock = this.clone();
parsedBlock.modifyType = undefined;
parsedBlock.optionUnitBits &= ~this.REDIRECT_BIT;
@ -3943,11 +3952,6 @@ FilterContainer.prototype.prime = function() {
keyvalStore.getItem('SNFE.destHNTrieContainer.trieDetails')
);
bidiTriePrime();
// Remove entries with obsolete name.
// TODO: Remove before publishing 1.41.0
keyvalStore.removeItem('SNFE.filterOrigin.trieDetails');
keyvalStore.removeItem('SNFE.FilterHostnameDict.trieDetails');
keyvalStore.removeItem('SNFE.filterDocOrigin.trieDetails');
};
/******************************************************************************/
@ -4741,8 +4745,8 @@ FilterContainer.prototype.unserialize = async function(s) {
/******************************************************************************/
FilterContainer.prototype.createCompiler = function(parser) {
return new FilterCompiler(parser);
FilterContainer.prototype.createCompiler = function() {
return new FilterCompiler();
};
/******************************************************************************/
@ -4768,7 +4772,7 @@ FilterContainer.prototype.fromCompiled = function(reader) {
FilterContainer.prototype.matchAndFetchModifiers = function(
fctxt,
modifierType
modifierName
) {
const typeBits = typeNameToTypeValue[fctxt.type] || otherTypeBitValue;
@ -4811,7 +4815,7 @@ FilterContainer.prototype.matchAndFetchModifiers = function(
const results = [];
const env = {
type: StaticFilteringParser.netOptionTokenIds.get(modifierType) || 0,
type: modifierTypeFromName.get(modifierName) || 0,
bits: 0,
th: 0,
iunit: 0,
@ -5223,7 +5227,7 @@ FilterContainer.prototype.redirectRequest = function(redirectEngine, fctxt) {
function parseRedirectRequestValue(directive) {
if ( directive.cache === null ) {
directive.cache =
StaticFilteringParser.parseRedirectValue(directive.value);
sfp.parseRedirectValue(directive.value);
}
return directive.cache;
}
@ -5336,7 +5340,7 @@ FilterContainer.prototype.filterQuery = function(fctxt) {
function parseQueryPruneValue(directive) {
if ( directive.cache === null ) {
directive.cache =
StaticFilteringParser.parseQueryPruneValue(directive.value);
sfp.parseQueryPruneValue(directive.value);
}
return directive.cache;
}

28
src/js/storage.js
View File

@ -40,8 +40,8 @@ import { hostnameFromURI } from './uri-utils.js';
import { i18n, i18n$ } from './i18n.js';
import { redirectEngine } from './redirect-engine.js';
import { sparseBase64 } from './base64-custom.js';
import { StaticFilteringParser } from './static-filtering-parser.js';
import { ubolog, ubologSet } from './console.js';
import * as sfp from './static-filtering-parser.js';
import {
permanentFirewall,
@ -1007,20 +1007,16 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
const expertMode =
details.assetKey !== this.userFiltersPath ||
this.hiddenSettings.filterAuthorMode !== false;
// Useful references:
// https://adblockplus.org/en/filter-cheatsheet
// https://adblockplus.org/en/filters
const parser = new StaticFilteringParser({
const parser = new sfp.AstFilterParser({
expertMode,
nativeCssHas: vAPI.webextFlavor.env.includes('native_css_has'),
maxTokenLength: staticNetFilteringEngine.MAX_TOKEN_LENGTH,
});
const compiler = staticNetFilteringEngine.createCompiler(parser);
const lineIter = new LineIterator(
parser.utils.preparser.prune(rawText, vAPI.webextFlavor.env)
sfp.utils.preparser.prune(rawText, vAPI.webextFlavor.env)
);
parser.setMaxTokenLength(staticNetFilteringEngine.MAX_TOKEN_LENGTH);
compiler.start(writer);
while ( lineIter.eot() === false ) {
@ -1031,23 +1027,19 @@ self.addEventListener('hiddenSettingsChanged', ( ) => {
line = line.slice(0, -2).trim() + lineIter.next().trim();
}
parser.analyze(line);
parser.parse(line);
if ( parser.shouldIgnore() ) { continue; }
if ( parser.isFilter() === false ) { continue; }
if ( parser.hasError() ) { continue; }
if ( parser.category === parser.CATStaticExtFilter ) {
if ( parser.isExtendedFilter() ) {
staticExtFilteringEngine.compile(parser, writer);
continue;
}
if ( parser.category !== parser.CATStaticNetFilter ) { continue; }
if ( parser.isNetworkFilter() === false ) { continue; }
// https://github.com/gorhill/uBlock/issues/2599
// convert hostname to punycode if needed
if ( parser.patternHasUnicode() && parser.toASCII() === false ) {
continue;
}
if ( compiler.compile(writer) ) { continue; }
if ( compiler.compile(parser, writer) ) { continue; }
if ( compiler.error !== undefined ) {
logger.writeOne({
realm: 'message',