documentation about available methods. add tests

This commit is contained in:
Alexandre Storelli 2019-03-29 17:32:21 +01:00
parent 4b3f331a64
commit c8dcdba36e
4 changed files with 89 additions and 9 deletions

View File

@ -2,7 +2,7 @@
![Adblock Radio](https://www.adblockradio.com/assets/img/abr_buddha_v3_175.png)
An adblocker for live radio streams and podcasts. Machine learning meets Shazam.
A library to block ads on live radio streams and podcasts. Machine learning meets Shazam.
Engine of [AdblockRadio.com](https://www.adblockradio.com).
Demo standalone player [available here](https://github.com/adblockradio/buffer-player).
@ -194,6 +194,18 @@ Property|Description|Default
`name`|Name of the radio stream according to [radio-browser.info](http://www.radio-browser.info)|None
`file`|File to analyse (optional, analyse the live stream otherwise)|None
### Methods
Acoustic model and hotlist files are refreshed automatically on startup. If you plan to continuously run the algo for a long time, you can trigger manual updates. Note those methods are only available in live stream analysis mode.
Method|Parameters|Description
------|----------|-----------
`refreshPredictorMl`|None|Manually refresh the ML model (live stream only)
`refreshPredictorHotlist`|None|Manually refresh the hotlist DB (live stream only)
`refreshMetadata`|None|Manually refresh the [metadata scraper](https://github.com/adblockradio/webradio-metadata) (live stream only)
`stopDl`|None|Stop Adblock Radio (live stream only)
### Optional configuration
Properties marked with a `*` are meant to be used only with live radio stream analysis, not file analysis where they are ignored.

View File

@ -645,20 +645,40 @@ class Analyser extends Readable {
}
refreshPredictorMl() {
if (this.config.file || this.config.records) {
log.warn("updating ML model is not possible when analysing files. skip.");
return false;
}
this.predictor.refreshPredictorMl();
return true;
}
refreshPredictorHotlist() {
if (this.config.file || this.config.records) {
log.warn("updating hotlist DB is not possible when analysing files. skip.");
return false;
}
this.predictor.refreshPredictorHotlist();
return true;
}
refreshMetadata() {
if (this.config.file || this.config.records) {
log.warn("updating hotlist DB is not possible when analysing files. skip.");
return false;
}
this.predictor.refreshMetadata();
return true;
}
stopDl() {
if (this.config.file || this.config.records) {
log.warn("not possible (yet?) to stop the processing of files. please kill the process instead.");
return false;
}
if (this.modelUpdatesInterval) clearInterval(this.modelUpdatesInterval);
if (this.predictor) this.predictor.stop();
return true;
}
_read() {

View File

@ -25,6 +25,8 @@ if (cluster.isMaster) {
let timedOut = false;
let fileOutput = {};
let fileOutputIsSane = false;
let refreshCorrectlyHandled = false;
let refreshError = false;
const timer = setTimeout(function() {
log.error('analysis timed out or was too slow. kill it.');
@ -44,6 +46,12 @@ if (cluster.isMaster) {
if (msg.data.blocksCleaned) {
gotBlocks = true;
}
} else if (msg.type === 'refresh') {
if (msg.hasError) {
refreshError = true;
}
refreshCorrectlyHandled = msg.result === false;
} else if (msg.type === 'end') {
finished = true;
}
@ -90,6 +98,11 @@ if (cluster.isMaster) {
assert(finished);
});
it("should reject attempts to reload ML model, hotlist DB or metadata scraper during analysis.", function() {
assert.equal(refreshError, false);
assert(refreshCorrectlyHandled);
});
it("should not have thrown errors", function() {
assert(!hasErrors);
});
@ -122,7 +135,7 @@ if (cluster.isMaster) {
assert(p);
if (TEST_ML) {
assert(p.gain > 20 && p.gain < 100);
assert(p.gain > 0 && p.gain < 200);
assert(p.ml);
assert(['0-ads', '1-speech', '2-music', '9-unsure'].includes(p.ml.class));
assert(p.ml.softmaxraw);
@ -181,6 +194,19 @@ if (cluster.isMaster) {
}
});
try {
const result = abr.refreshPredictorMl() ||
abr.refreshPredictorHotlist() ||
abr.refreshMetadata() ||
abr.stopDl();
process.send({ type: 'refresh', result, hasError: false });
log.info('refresh attempted. result=' + result);
} catch (e) {
log.error('refresh attempt error: ' + e);
process.send({ type: 'refresh', hasError: true });
}
abr.on("data", function(obj) {
//log.info(JSON.stringify(obj, null, "\t"));
process.send({ type: 'data', data: obj });

View File

@ -14,6 +14,7 @@ const MLJS = process.argv.includes('--mljs');
if (cluster.isMaster) {
const REFRESH_DELAY = 8000;
const CLOSE_DELAY = 15000;
const TIMEOUT = 10000; // ms counted in addition to CLOSE_DELAY
@ -26,11 +27,20 @@ if (cluster.isMaster) {
let exitCode = null;
let timeout = null;
let timedOut = false;
let refreshRequested = false;
let refreshOK = false;
let metaFiles = [];
let metaFilesContent = null;
let metaFilesSane = null;
setTimeout(function() {
log.info('refresh ML, hotlist and metadata modules');
refreshRequested = true;
cp.send({ action: 'refresh' });
}, REFRESH_DELAY);
setTimeout(function() {
log.info('stop the stream analysis');
stopped = true;
@ -56,6 +66,8 @@ if (cluster.isMaster) {
if (!metaFiles.includes(msg.data.metadataPath)) metaFiles.push(msg.data.metadataPath);
}
}
} else if (msg.type === 'refresh') {
refreshOK = !msg.hasError;
} else if (msg.type === 'end') {
finished = true;
}
@ -144,7 +156,7 @@ if (cluster.isMaster) {
// ML module is usually not ready at startup of live stream analysis
if (TEST_ML && p.ml) {
assert(p.gain > 20 && p.gain < 100);
assert(p.gain > 0 && p.gain < 200);
assert(p.ml);
assert(['0-ads', '1-speech', '2-music', '9-unsure'].includes(p.ml.class));
assert(p.ml.softmaxraw);
@ -179,11 +191,10 @@ if (cluster.isMaster) {
}
});
it("should refresh ML model when requested");
it("should refresh hotlist db when requested");
it("should refresh metadata parser when requested");
it("should refresh ML model, hotlist and metadata when requested", function() {
assert(refreshRequested);
assert(refreshOK);
});
});
} else {
@ -207,7 +218,7 @@ if (cluster.isMaster) {
abr.on("data", function(obj) {
obj.liveResult.audio = "[redacted]";
//log.info(obj.metadataPath);
log.info(JSON.stringify(obj.liveResult, null, "\t"));
//log.info(JSON.stringify(obj.liveResult, null, "\t"));
process.send({ type: 'data', data: obj });
});
@ -220,6 +231,17 @@ if (cluster.isMaster) {
process.on('message', function(msg) {
if (msg && msg.action === 'stop') {
abr.stopDl();
} else if (msg && msg.action === 'refresh') {
try {
abr.refreshPredictorMl();
abr.refreshPredictorHotlist();
abr.refreshMetadata();
process.send({ type: 'refresh', hasError: false });
log.info('refresh OK');
} catch (e) {
log.error('refresh error: ' + e);
process.send({ type: 'refresh', hasError: true });
}
}
});
}