Add support for nodejs flavors

The main nodejs flavor is "npm", which is to be used to
lint/test and the publication of an official npm
package -- and by design it has dependencies on mocha,
eslint, etc.

A new flavor "dig" has been created with minimal
dependencies and which purpose is to easily allow to
write specialized code to investigate local code changes
in uBO -- and it's not meant for publication.

Consequently, "make nodejs" has been replaced with
"make npm", and a new "dig" target has been added to the
makefile, to be used for instrumenting local code changes
for investigation purpose.
This commit is contained in:
Raymond Hill 2021-08-15 10:43:36 -04:00
parent 0a11999459
commit 087da6407d
No known key found for this signature in database
GPG Key ID: 25E1490B761470C2
13 changed files with 152 additions and 226 deletions

View File

@ -39,7 +39,7 @@ jobs:
tools/make-chromium.sh ${{ steps.release_info.outputs.VERSION }}
tools/make-firefox.sh ${{ steps.release_info.outputs.VERSION }}
tools/make-thunderbird.sh ${{ steps.release_info.outputs.VERSION }}
tools/make-nodejs.sh ${{ steps.release_info.outputs.VERSION }}
tools/make-npm.sh ${{ steps.release_info.outputs.VERSION }}
- name: Upload Chromium package
uses: actions/upload-release-asset@v1
env:
@ -73,6 +73,6 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
upload_url: ${{ steps.create_release.outputs.upload_url }}
asset_path: dist/build/uBlock0_${{ steps.release_info.outputs.VERSION }}.nodejs.tgz
asset_name: uBlock0_${{ steps.release_info.outputs.VERSION }}.nodejs.tgz
asset_path: dist/build/uBlock0_${{ steps.release_info.outputs.VERSION }}.npm.tgz
asset_name: uBlock0_${{ steps.release_info.outputs.VERSION }}.npm.tgz
asset_content_type: application/octet-stream

View File

@ -1,4 +1,4 @@
.PHONY: all clean test lint chromium firefox nodejs
.PHONY: all clean test lint chromium firefox npm
sources := $(wildcard src/* src/*/* src/*/*/* src/*/*/*/*)
platform := $(wildcard platform/* platform/*/*)
@ -7,7 +7,7 @@ assets := $(wildcard submodules/uAssets/* \
submodules/uAssets/*/*/* \
submodules/uAssets/*/*/*/*)
all: chromium firefox nodejs
all: chromium firefox npm
dist/build/uBlock0.chromium: tools/make-chromium.sh $(sources) $(platform) $(assets)
tools/make-chromium.sh
@ -21,17 +21,23 @@ dist/build/uBlock0.firefox: tools/make-firefox.sh $(sources) $(platform) $(asset
# Build the extension for Firefox.
firefox: dist/build/uBlock0.firefox
dist/build/uBlock0.nodejs: tools/make-nodejs.sh $(sources) $(platform) $(assets)
tools/make-nodejs.sh
dist/build/uBlock0.npm: tools/make-nodejs.sh $(sources) $(platform) $(assets)
tools/make-npm.sh
# Build the Node.js package.
nodejs: dist/build/uBlock0.nodejs
npm: dist/build/uBlock0.npm
lint: nodejs
cd dist/build/uBlock0.nodejs && npm install && npm run lint
lint: npm
cd dist/build/uBlock0.npm && npm install && npm run lint
test: nodejs
cd dist/build/uBlock0.nodejs && npm install && npm run test
test: npm
cd dist/build/uBlock0.npm && npm install && npm run test
dist/build/uBlock0.dig: tools/make-nodejs.sh $(sources) $(platform) $(assets)
tools/make-dig.sh
dig: dist/build/uBlock0.dig
cd dist/build/uBlock0.dig && npm install && npm run test
# Update submodules.
update-submodules:

View File

@ -102,7 +102,7 @@ and use this capability now if you figure out the details.
You can directly use specific APIs exposed by this package, here are some of
them, which are used internally by uBO's SNFE.
### `HNTrieContainer`
### HNTrieContainer
A well optimised [compressed trie](https://en.wikipedia.org/wiki/Trie#Compressing_tries)
container specialized to specifically store and lookup hostnames.

View File

@ -95,8 +95,7 @@ function pslInit(raw) {
return globals.publicSuffixList;
}
const require = createRequire(import.meta.url); // jshint ignore:line
raw = require('./data/effective_tld_names.json');
raw = readFileSync(resolve(__dirname, './assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat'), 'utf8')
if ( typeof raw !== 'string' || raw.trim() === '' ) {
console.error('Unable to populate public suffix list');
return;
@ -203,14 +202,14 @@ class MockStorage {
}
const fctx = new FilteringContext();
let snfeInstance = null;
let snfeProxyInstance = null;
class StaticNetFilteringEngine {
constructor() {
if ( snfeInstance !== null ) {
if ( snfeProxyInstance !== null ) {
throw new Error('Only a single instance is supported.');
}
snfeInstance = this;
snfeProxyInstance = this;
}
async useLists(lists) {
@ -253,6 +252,11 @@ class StaticNetFilteringEngine {
return instance;
}
static release() {
useLists([]);
snfeProxyInstance = null;
}
}
/******************************************************************************/

View File

@ -1,160 +0,0 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
Copyright (C) 2014-present Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see {http://www.gnu.org/licenses/}.
Home: https://github.com/gorhill/uBlock
*/
/* eslint-disable-next-line no-redeclare */
/* globals process */
'use strict';
/******************************************************************************/
import { strict as assert } from 'assert';
import { spawn } from "child_process";
import { createRequire } from 'module';
import { promisify } from 'util';
import {
enableWASM,
StaticNetFilteringEngine,
} from './index.js';
import HNTrieContainer from './js/hntrie.js';
/******************************************************************************/
function fetch(listName) {
return new Promise(resolve => {
const require = createRequire(import.meta.url); // jshint ignore:line
resolve(require(`./data/${listName}.json`));
});
}
function testSNFE(engine) {
let result = 0;
// Tests
// Not blocked
result = engine.matchRequest({
originURL: 'https://www.bloomberg.com/',
url: 'https://www.bloomberg.com/tophat/assets/v2.6.1/that.css',
type: 'stylesheet'
});
if ( result !== 0 ) {
console.log(engine.toLogData());
}
// Blocked
result = engine.matchRequest({
originURL: 'https://www.bloomberg.com/',
url: 'https://securepubads.g.doubleclick.net/tag/js/gpt.js',
type: 'script'
});
if ( result !== 0 ) {
console.log(engine.toLogData());
}
// Unblocked
result = engine.matchRequest({
originURL: 'https://www.bloomberg.com/',
url: 'https://sourcepointcmp.bloomberg.com/ccpa.js',
type: 'script'
});
if ( result !== 0 ) {
console.log(engine.toLogData());
}
}
async function doSNFE() {
const engine = await StaticNetFilteringEngine.create();
await engine.useLists([
fetch('easylist').then(raw => ({ name: 'easylist', raw })),
fetch('easyprivacy').then(raw => ({ name: 'easyprivacy', raw })),
]);
testSNFE(engine);
const serialized = await engine.serialize();
engine.useLists([]);
assert.notDeepEqual(await engine.serialize(), serialized);
testSNFE(engine);
await engine.deserialize(serialized);
assert.deepEqual(await engine.serialize(), serialized);
testSNFE(engine);
}
async function doHNTrie() {
const trieContainer = new HNTrieContainer();
const aTrie = trieContainer.createOne();
aTrie.add('example.org');
aTrie.add('example.com');
const anotherTrie = trieContainer.createOne();
anotherTrie.add('foo.invalid');
anotherTrie.add('bar.invalid');
// matches() return the position at which the match starts, or -1 when
// there is no match.
// Matches: return 4
console.log("aTrie.matches('www.example.org')", aTrie.matches('www.example.org'));
// Does not match: return -1
console.log("aTrie.matches('www.foo.invalid')", aTrie.matches('www.foo.invalid'));
// Does not match: return -1
console.log("anotherTrie.matches('www.example.org')", anotherTrie.matches('www.example.org'));
// Matches: return 0
console.log("anotherTrie.matches('foo.invalid')", anotherTrie.matches('foo.invalid'));
}
async function spawnMocha() {
await promisify(spawn)('mocha', [ '--experimental-vm-modules', '--no-warnings', 'tests' ], { stdio: [ 'inherit', 'inherit', 'inherit' ] });
}
async function main() {
try {
const result = await enableWASM();
if ( result !== true ) {
console.log('Failed to enable all WASM code paths');
}
} catch(ex) {
console.log(ex);
}
await doSNFE();
await doHNTrie();
if ( process.argv[2] === '--mocha' ) {
await spawnMocha();
}
}
main();
/******************************************************************************/

46
platform/npm/test.js Normal file
View File

@ -0,0 +1,46 @@
/*******************************************************************************
uBlock Origin - a browser extension to block requests.
Copyright (C) 2014-present Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see {http://www.gnu.org/licenses/}.
Home: https://github.com/gorhill/uBlock
*/
/* eslint-disable-next-line no-redeclare */
/* globals process */
'use strict';
/******************************************************************************/
import { spawn } from "child_process";
import { promisify } from 'util';
/******************************************************************************/
async function spawnMocha() {
await promisify(spawn)('mocha', [ '--experimental-vm-modules', '--no-warnings', 'tests' ], { stdio: [ 'inherit', 'inherit', 'inherit' ] });
}
async function main() {
if ( process.argv[2] === '--mocha' ) {
await spawnMocha();
}
}
main();
/******************************************************************************/

17
tools/make-dig.sh Executable file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env bash
#
# This script assumes a linux environment
set -e
DES="dist/build/uBlock0.dig"
./tools/make-nodejs.sh $DES
./tools/make-assets.sh $DES
cp -R platform/dig/* $DES/
cd $DES
npm run build
echo "*** uBlock0.dig: Package done."

View File

@ -4,18 +4,7 @@
set -e
DES=dist/build/uBlock0.nodejs
TMPDIR=tmp
mkdir -p $TMPDIR
# Save existing npm dependencies if present so that we do not have to fetch
# them all again
if [ -d "$DES/node_modules" ]; then
mv "$DES/node_modules" "$TMPDIR/node_modules"
fi
rm -rf $DES
DES=$1
mkdir -p $DES/js
cp src/js/base64-custom.js $DES/js
@ -47,42 +36,6 @@ node -pe "JSON.stringify(Array.from(fs.readFileSync('src/js/wasm/biditrie.wasm')
node -pe "JSON.stringify(Array.from(fs.readFileSync('src/lib/publicsuffixlist/wasm/publicsuffixlist.wasm')))" \
> $DES/lib/publicsuffixlist/wasm/publicsuffixlist.wasm.json
git submodule update --depth 1 --init
UASSETS=submodules/uAssets
# https://github.com/uBlockOrigin/uBlock-issues/issues/1664#issuecomment-888332409
THIRDPARTY=$UASSETS/thirdparties/publicsuffix.org
mkdir -p $DES/data
node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/list/effective_tld_names.dat', 'utf8'))" \
> $DES/data/effective_tld_names.json
THIRDPARTY=$UASSETS/thirdparties/easylist-downloads.adblockplus.org
node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/easylist.txt', 'utf8'))" \
> $DES/data/easylist.json
node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/easyprivacy.txt', 'utf8'))" \
> $DES/data/easyprivacy.json
cp platform/nodejs/.*.json $DES/
cp platform/nodejs/*.js $DES/
cp platform/nodejs/*.json $DES/
cp platform/nodejs/README.md $DES/
cp LICENSE.txt $DES/
cp -R platform/nodejs/tests $DES/
cd $DES
npm run build
tarballname=$(npm pack 2> /dev/null)
if [ "$1" ]; then
echo "*** uBlock0.nodejs: Creating versioned package..."
mv $tarballname ../uBlock0_"$1".nodejs.tgz
else
echo "*** uBlock0.nodejs: Creating plain package..."
mv $tarballname ../uBlock0.nodejs.tgz
fi
cd -
# Restore saved npm dependencies
if [ -d "$TMPDIR/node_modules" ]; then
mv "$TMPDIR/node_modules" "$DES/node_modules"
fi
echo "*** uBlock0.nodejs: Package done."

60
tools/make-npm.sh Executable file
View File

@ -0,0 +1,60 @@
#!/usr/bin/env bash
#
# This script assumes a linux environment
set -e
DES="dist/build/uBlock0.npm"
TMPDIR=tmp
mkdir -p $TMPDIR
# Save existing npm dependencies if present so that we do not have to fetch
# them all again
if [ -d "$DES/node_modules" ]; then
mv "$DES/node_modules" "$TMPDIR/node_modules"
fi
rm -rf $DES
./tools/make-nodejs.sh $DES
UASSETS=submodules/uAssets
# https://github.com/uBlockOrigin/uBlock-issues/issues/1664#issuecomment-888332409
mkdir -p $DES/assets/thirdparties/publicsuffix.org/list
THIRDPARTY=$UASSETS/thirdparties/publicsuffix.org
node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/list/effective_tld_names.dat', 'utf8'))" \
> $DES/assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat
mkdir -p $DES/data
THIRDPARTY=$UASSETS/thirdparties/easylist-downloads.adblockplus.org
node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/easylist.txt', 'utf8'))" \
> $DES/data/easylist.json
node -pe "JSON.stringify(fs.readFileSync('$THIRDPARTY/easyprivacy.txt', 'utf8'))" \
> $DES/data/easyprivacy.json
# Target-specific
cp platform/npm/*.json $DES/
cp platform/npm/.*.json $DES/
cp platform/npm/*.js $DES/
cp -R platform/npm/tests $DES/
cd $DES
npm run build
tarballname=$(npm pack 2> /dev/null)
if [ "$1" ]; then
echo "*** uBlock0.npm: Creating versioned package..."
mv $tarballname ../uBlock0_$1.npm.tgz
else
echo "*** uBlock0.npm: Creating plain package..."
mv $tarballname ../uBlock0.npm.tgz
fi
cd -
# Restore saved npm dependencies
if [ -d "$TMPDIR/node_modules" ]; then
mv "$TMPDIR/node_modules" "$DES/node_modules"
fi
echo "*** uBlock0.npm: Package done."