opensourcemirror
/
adblockradio
mirror of https://github.com/adblockradio/adblockradio.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

new logging system

This commit is contained in:
Alexandre Storelli 2018-04-10 11:57:52 +02:00
parent 1b6bfe9dea
commit b8d2b1902a
8 changed files with 84 additions and 496 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -1,2 +1,7 @@
records/
node_modules/
log.js
log/
*.mp3
*.log
*.png

25
backend.js
View File

@ -1,9 +1,9 @@
var Db = require("./predictor-db/db.js");
var MlPredictor = require("./predictor-ml/ml.js");
var Dl = require("../adblockradio-dl/dl.js");
const Codegen = require("stream-audio-fingerprint");
var { StreamDl } = require("../adblockradio-dl/dl.js");
var { getMeta } = require("webradio-metadata");
var log = require("loglevel");
log.setLevel("debug");
var { log } = require("./log.js")("pred-master");
var country = "France";
var name = "RTL";
@ -14,7 +14,7 @@ const ENABLE_PREDICTOR_ML = false;
const SAVE_AUDIO = true;
const FETCH_METADATA = false;
var dl = new Dl({ country: country, name: name, segDuration: 10 });
var dl = new StreamDl({ country: country, name: name, segDuration: 10 });
dl.on("error", function(err) {
console.log("dl err=" + err);
});
@ -31,12 +31,11 @@ var decoder = require('child_process').spawn('ffmpeg', [
//dl.pipe(decoder.stdin); //.write(data);
if (ENABLE_PREDICTOR_FINGERPRINT) {
var Codegen = require("stream-audio-fingerprint");
var fingerprinter = new Codegen();
decoder.stdout.pipe(fingerprinter);
}
if (ENABLE_PREDICTOR_ML) {
var mlPredictor = new MlPredictor({ country: country, name: name});
var mlPredictor = new MlPredictor({ country: country, name: name });
decoder.stdout.pipe(mlPredictor);
}
@ -101,16 +100,10 @@ dl.on("metadata", function(metadata) {
if (ENABLE_PREDICTOR_ML) mlPredictor.pipe(dbs.metadata);
if (FETCH_METADATA) {
getMeta(country, name, function(err, parsedMeta, corsEnabled) {
if (err) {
log.warn("getMeta: error fetching title meta. err=" + err);
} else {
log.info(country + "_" + name + " meta=" + JSON.stringify(parsedMeta));
if (!dbs.metadata.ended) {
dbs.metadata.write({ type: "title", data: parsedMeta });
} else {
log.warn("getMeta: could not write metadata, stream already ended");
}
}
if (err) return log.warn("getMeta: error fetching title meta. err=" + err);
if (dbs.metadata.ended) return log.warn("getMeta: could not write metadata, stream already ended");
log.info(country + "_" + name + " meta=" + JSON.stringify(parsedMeta));
dbs.metadata.write({ type: "title", data: parsedMeta });
});
}
decoder.stdin.write(dataObj.data);

0
log/.gitkeep Normal file
View File

519
package-lock.json generated
View File

@ -4,20 +4,14 @@
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"accepts": {
"version": "1.3.4",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.4.tgz",
"integrity": "sha1-hiRnWMfdbSGmR0/whKR0DsBesh8=",
"ansi-styles": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz",
"integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==",
"requires": {
"mime-types": "2.1.17",
"negotiator": "0.6.1"
"color-convert": "1.9.1"
}
},
"array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
},
"async": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz",
@ -26,296 +20,61 @@
"lodash": "4.17.4"
}
},
"body-parser": {
"version": "1.18.2",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz",
"integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=",
"chalk": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz",
"integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==",
"requires": {
"bytes": "3.0.0",
"content-type": "1.0.4",
"debug": "2.6.9",
"depd": "1.1.1",
"http-errors": "1.6.2",
"iconv-lite": "0.4.19",
"on-finished": "2.3.0",
"qs": "6.5.1",
"raw-body": "2.3.2",
"type-is": "1.6.15"
"ansi-styles": "3.2.1",
"escape-string-regexp": "1.0.5",
"supports-color": "5.3.0"
}
},
"bytes": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
"integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
},
"camelize": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/camelize/-/camelize-1.0.0.tgz",
"integrity": "sha1-FkpUg+Yw+kMh5a8HAg5TGDGyYJs="
},
"chunked-stream": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/chunked-stream/-/chunked-stream-0.0.2.tgz",
"integrity": "sha1-zPdQmNrb8Ls4/zB+EJWCeAUHk54="
},
"content-disposition": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz",
"integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ="
},
"content-security-policy-builder": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/content-security-policy-builder/-/content-security-policy-builder-1.1.0.tgz",
"integrity": "sha1-2R8bB2I2wRmFDH3umSS/VeBXcrM=",
"color-convert": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz",
"integrity": "sha512-mjGanIiwQJskCC18rPR6OmrZ6fm2Lc7PeGFYwCmy5J34wC6F1PzdGL6xeMfmgicfYcNLGuVFA3WzXtIDCQSZxQ==",
"requires": {
"dashify": "0.2.2"
"color-name": "1.1.3"
}
},
"content-type": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
"integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
},
"cookie": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz",
"integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s="
},
"cookie-signature": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
"integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
},
"dasherize": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/dasherize/-/dasherize-2.0.0.tgz",
"integrity": "sha1-bYCcnNDPe7iVLYD8hPoT1H3bEwg="
},
"dashify": {
"version": "0.2.2",
"resolved": "https://registry.npmjs.org/dashify/-/dashify-0.2.2.tgz",
"integrity": "sha1-agdBWgHJH69KMuONnfunH2HLIP4="
},
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
}
},
"depd": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz",
"integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k="
},
"destroy": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
"integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
},
"dns-prefetch-control": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/dns-prefetch-control/-/dns-prefetch-control-0.1.0.tgz",
"integrity": "sha1-YN20V3dOF48flBXwyrsOhbCzALI="
},
"dont-sniff-mimetype": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/dont-sniff-mimetype/-/dont-sniff-mimetype-1.0.0.tgz",
"integrity": "sha1-WTKJDcn04vGeXrAqIAJuXl78j1g="
"color-name": {
"version": "1.1.3",
"resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz",
"integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU="
},
"dsp.js": {
"version": "git+https://git@github.com/corbanbrook/dsp.js.git#52d60e4ece9158b2dac9261790d45421a1556cae"
"version": "git+https://git@github.com/corbanbrook/dsp.js.git#051f341d8e1d05515efecb1851bbc862decf02bc"
},
"ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
"escape-string-regexp": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz",
"integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ="
},
"encodeurl": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.1.tgz",
"integrity": "sha1-eePVhlU0aQn+bw9Fpd5oEDspTSA="
},
"escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
"integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
},
"etag": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
"integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
},
"expect-ct": {
"version": "0.1.0",
"resolved": "https://registry.npmjs.org/expect-ct/-/expect-ct-0.1.0.tgz",
"integrity": "sha1-UnNWeN4YUwiQ2Ne5XwrGNkCVgJQ="
},
"express": {
"version": "4.16.2",
"resolved": "https://registry.npmjs.org/express/-/express-4.16.2.tgz",
"integrity": "sha1-41xt/i1kt9ygpc1PIXgb4ymeB2w=",
"requires": {
"accepts": "1.3.4",
"array-flatten": "1.1.1",
"body-parser": "1.18.2",
"content-disposition": "0.5.2",
"content-type": "1.0.4",
"cookie": "0.3.1",
"cookie-signature": "1.0.6",
"debug": "2.6.9",
"depd": "1.1.1",
"encodeurl": "1.0.1",
"escape-html": "1.0.3",
"etag": "1.8.1",
"finalhandler": "1.1.0",
"fresh": "0.5.2",
"merge-descriptors": "1.0.1",
"methods": "1.1.2",
"on-finished": "2.3.0",
"parseurl": "1.3.2",
"path-to-regexp": "0.1.7",
"proxy-addr": "2.0.2",
"qs": "6.5.1",
"range-parser": "1.2.0",
"safe-buffer": "5.1.1",
"send": "0.16.1",
"serve-static": "1.13.1",
"setprototypeof": "1.1.0",
"statuses": "1.3.1",
"type-is": "1.6.15",
"utils-merge": "1.0.1",
"vary": "1.1.2"
}
},
"finalhandler": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.0.tgz",
"integrity": "sha1-zgtoVbRYU+eRsvzGgARtiCU91/U=",
"requires": {
"debug": "2.6.9",
"encodeurl": "1.0.1",
"escape-html": "1.0.3",
"on-finished": "2.3.0",
"parseurl": "1.3.2",
"statuses": "1.3.1",
"unpipe": "1.0.0"
}
},
"forwarded": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
"integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
},
"frameguard": {
"has-flag": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/frameguard/-/frameguard-3.0.0.tgz",
"integrity": "sha1-e8rUae57lukdEs6zlZx4I1qScuk="
},
"fresh": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
"integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
},
"helmet": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/helmet/-/helmet-3.9.0.tgz",
"integrity": "sha512-czCyS77TyanWlfVSoGlb9GBJV2Q2zJayKxU5uBw0N1TzDTs/qVNh1SL8Q688KU0i0Sb7lQ/oLtnaEqXzl2yWvA==",
"requires": {
"dns-prefetch-control": "0.1.0",
"dont-sniff-mimetype": "1.0.0",
"expect-ct": "0.1.0",
"frameguard": "3.0.0",
"helmet-csp": "2.6.0",
"hide-powered-by": "1.0.0",
"hpkp": "2.0.0",
"hsts": "2.1.0",
"ienoopen": "1.0.0",
"nocache": "2.0.0",
"referrer-policy": "1.1.0",
"x-xss-protection": "1.0.0"
}
},
"helmet-csp": {
"version": "2.6.0",
"resolved": "https://registry.npmjs.org/helmet-csp/-/helmet-csp-2.6.0.tgz",
"integrity": "sha512-n/oW9l6RtO4f9YvphsNzdvk1zITrSN7iRT8ojgrJu/N3mVdHl9zE4OjbiHWcR64JK32kbqx90/yshWGXcjUEhw==",
"requires": {
"camelize": "1.0.0",
"content-security-policy-builder": "1.1.0",
"dasherize": "2.0.0",
"lodash.reduce": "4.6.0",
"platform": "1.3.4"
}
},
"hide-powered-by": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/hide-powered-by/-/hide-powered-by-1.0.0.tgz",
"integrity": "sha1-SoWtZYgfYoV/xwr3F0oRhNzM4ys="
},
"hpkp": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/hpkp/-/hpkp-2.0.0.tgz",
"integrity": "sha1-EOFCJk52IVpdMMROxD3mTe5tFnI="
},
"hsts": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/hsts/-/hsts-2.1.0.tgz",
"integrity": "sha512-zXhh/DqgrTXJ7erTN6Fh5k/xjMhDGXCqdYN3wvxUvGUQvnxcFfUd8E+6vLg/nk3ss1TYMb+DhRl25fYABioTvA=="
},
"http-errors": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz",
"integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=",
"requires": {
"depd": "1.1.1",
"inherits": "2.0.3",
"setprototypeof": "1.0.3",
"statuses": "1.3.1"
},
"dependencies": {
"setprototypeof": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz",
"integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ="
}
}
},
"iconv-lite": {
"version": "0.4.19",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz",
"integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ=="
},
"ienoopen": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/ienoopen/-/ienoopen-1.0.0.tgz",
"integrity": "sha1-NGpCj0dKrI9QzzeE6i0PFvYr2ms="
},
"inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
},
"ipaddr.js": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.5.2.tgz",
"integrity": "sha1-1LUFvemUaYfM8PxY2QEP+WB+P6A="
"resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
"integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0="
},
"lodash": {
"version": "4.17.4",
"resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz",
"integrity": "sha1-eCA6TRwyiuHYbcpkYONptX9AVa4="
},
"lodash.reduce": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/lodash.reduce/-/lodash.reduce-4.6.0.tgz",
"integrity": "sha1-8atrg5KZrUj3hKu/R2WW8DuRTTs="
},
"loglevel": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.0.tgz",
"integrity": "sha1-rgyqVhERSYxboTcj1vtjHSQAOTQ="
"version": "1.6.1",
"resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.1.tgz",
"integrity": "sha1-4PyVEztu8nbNyIh82vJKpvFW+Po="
},
"loglevel-plugin-prefix": {
"version": "0.8.3",
"resolved": "https://registry.npmjs.org/loglevel-plugin-prefix/-/loglevel-plugin-prefix-0.8.3.tgz",
"integrity": "sha512-oFDEE3krjFTlyXfdT6shN4HsIlDrbPyleI2WIgRfn//oUEVfe8dP7goO9ktTSdOQC08CTKshKHHdZXe4Dy9TxQ=="
},
"m3u8": {
"version": "0.0.7",
@ -325,168 +84,21 @@
"chunked-stream": "0.0.2"
}
},
"media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
"integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
},
"merge-descriptors": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
"integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
},
"methods": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
"integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
},
"mime": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz",
"integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ=="
},
"mime-db": {
"version": "1.30.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.30.0.tgz",
"integrity": "sha1-dMZD2i3Z1qRTmZY0ZbJtXKfXHwE="
},
"mime-types": {
"version": "2.1.17",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.17.tgz",
"integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=",
"requires": {
"mime-db": "1.30.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"nan": {
"version": "2.7.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.7.0.tgz",
"integrity": "sha1-2Vv3IeyHfgjbJ27T/G63j5CDrUY="
},
"negotiator": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz",
"integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk="
},
"nocache": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/nocache/-/nocache-2.0.0.tgz",
"integrity": "sha1-ICtIAhoMTL3i34DeFaF0Q8i0OYA="
},
"node-png": {
"version": "0.4.3",
"resolved": "https://registry.npmjs.org/node-png/-/node-png-0.4.3.tgz",
"integrity": "sha1-RQIjeWuC08yg/+Sl1cf6l0hZdOc="
},
"on-finished": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
"integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=",
"requires": {
"ee-first": "1.1.1"
}
},
"parseurl": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz",
"integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M="
},
"path-to-regexp": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
},
"platform": {
"version": "1.3.4",
"resolved": "https://registry.npmjs.org/platform/-/platform-1.3.4.tgz",
"integrity": "sha1-bw+xftqqSPIUQrOpdcBjEw8cPr0="
},
"proxy-addr": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.2.tgz",
"integrity": "sha1-ZXFQT0e7mI7IGAJT+F3X4UlSvew=",
"requires": {
"forwarded": "0.1.2",
"ipaddr.js": "1.5.2"
}
},
"qs": {
"version": "6.5.1",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz",
"integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A=="
},
"range-parser": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz",
"integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4="
},
"raw-body": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz",
"integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=",
"requires": {
"bytes": "3.0.0",
"http-errors": "1.6.2",
"iconv-lite": "0.4.19",
"unpipe": "1.0.0"
}
},
"referrer-policy": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.1.0.tgz",
"integrity": "sha1-NXdOtzW/UPtsB46DM0tHI1AgfXk="
},
"safe-buffer": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
"integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
},
"sax": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
},
"send": {
"version": "0.16.1",
"resolved": "https://registry.npmjs.org/send/-/send-0.16.1.tgz",
"integrity": "sha512-ElCLJdJIKPk6ux/Hocwhk7NFHpI3pVm/IZOYWqUmoxcgeyM+MpxHHKhb8QmlJDX1pU6WrgaHBkVNm73Sv7uc2A==",
"requires": {
"debug": "2.6.9",
"depd": "1.1.1",
"destroy": "1.0.4",
"encodeurl": "1.0.1",
"escape-html": "1.0.3",
"etag": "1.8.1",
"fresh": "0.5.2",
"http-errors": "1.6.2",
"mime": "1.4.1",
"ms": "2.0.0",
"on-finished": "2.3.0",
"range-parser": "1.2.0",
"statuses": "1.3.1"
}
},
"serve-static": {
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.1.tgz",
"integrity": "sha512-hSMUZrsPa/I09VYFJwa627JJkNs0NrfL1Uzuup+GqHfToR2KcsXFymXSV90hoyw3M+msjFuQly+YzIH/q0MGlQ==",
"requires": {
"encodeurl": "1.0.1",
"escape-html": "1.0.3",
"parseurl": "1.3.2",
"send": "0.16.1"
}
},
"setprototypeof": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz",
"integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ=="
},
"sqlite3": {
"version": "3.1.13",
"resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-3.1.13.tgz",
@ -1199,72 +811,47 @@
}
}
},
"statuses": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz",
"integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4="
},
"stream-audio-fingerprint": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/stream-audio-fingerprint/-/stream-audio-fingerprint-1.0.1.tgz",
"integrity": "sha512-16YwNlkjLVr7SfO+vZBisJ0cQ5XAy0tVN6HAlv5N6FpY5HPMHo0cM3L/B67NAevNCTjqUH8JbNQeSy2aaeugaQ==",
"requires": {
"dsp.js": "git+https://git@github.com/corbanbrook/dsp.js.git#52d60e4ece9158b2dac9261790d45421a1556cae",
"dsp.js": "git+https://git@github.com/corbanbrook/dsp.js.git#051f341d8e1d05515efecb1851bbc862decf02bc",
"node-png": "0.4.3"
}
},
"type-is": {
"version": "1.6.15",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.15.tgz",
"integrity": "sha1-yrEPtJCeRByChC6v4a1kbIGARBA=",
"supports-color": {
"version": "5.3.0",
"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz",
"integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==",
"requires": {
"media-typer": "0.3.0",
"mime-types": "2.1.17"
"has-flag": "3.0.0"
}
},
"unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
},
"utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
"integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
},
"vary": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
"integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
},
"webradio-metadata": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/webradio-metadata/-/webradio-metadata-0.1.3.tgz",
"integrity": "sha512-X+9q4IdO7Kbf4JN+V/+RSnO0XaBW+VAxGssyYlni4j9AnfpQ4ilWJiqeWdrkjXHXRikATDbvKBgYHlHZBd86sg==",
"version": "0.1.15",
"resolved": "https://registry.npmjs.org/webradio-metadata/-/webradio-metadata-0.1.15.tgz",
"integrity": "sha512-Tuue71q1aEZg2Bls0LoWi3SlSMRJuF+uF7Z/e7FducahMWeIbKSNyl8mZjY76D9KQCOl9XdXxvmcbMO6Zt2DDA==",
"requires": {
"express": "4.16.2",
"helmet": "3.9.0",
"chalk": "2.3.2",
"loglevel": "1.6.1",
"loglevel-plugin-prefix": "0.8.3",
"xml2js": "0.4.19"
}
},
"x-xss-protection": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/x-xss-protection/-/x-xss-protection-1.0.0.tgz",
"integrity": "sha1-iYr7k4abJGYc+cUvnujbjtB2Tdk="
},
"xml2js": {
"version": "0.4.19",
"resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz",
"integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==",
"requires": {
"sax": "1.2.4",
"xmlbuilder": "9.0.4"
"xmlbuilder": "9.0.7"
}
},
"xmlbuilder": {
"version": "9.0.4",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.4.tgz",
"integrity": "sha1-UZy0ymhtAFqEINNJbz8MruzKWA8="
"version": "9.0.7",
"resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz",
"integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0="
}
}
}

6
package.json
View File

@ -17,11 +17,11 @@
"dependencies": {
"async": "^2.6.0",
"dsp.js": "git+https://git@github.com/corbanbrook/dsp.js.git",
"loglevel": "^1.6.0",
"loglevel": "^1.6.1",
"m3u8": "0.0.7",
"node-png": "^0.4.3",
"sqlite3": "^3.1.13",
"stream-audio-fingerprint": "^1.0.1",
"webradio-metadata": "0.1.3"
"stream-audio-fingerprint": "1.0.x",
"webradio-metadata": "^0.1.15"
}
}

3
predictor-db/db.js
View File

@ -1,8 +1,7 @@
var sqlite3 = require("sqlite3").verbose();
var { Writable, Transform } = require("stream");
var fs = require("fs");
var log = require("loglevel");
log.setLevel("debug");
var { log } = require("../log.js")("pred-db");
var cp = require("child_process");
var async = require("async");
var findDataFiles = require("./findDataFiles.js");

17
predictor-db/findDataFiles.js
View File

@ -1,6 +1,5 @@
var fs = require("fs");
var log = require("loglevel");
log.setLevel("debug");
const { log } = require("../log.js")("pred-db-findDataFiles");
var async = require("async");
var consts = {
WLARRAY: ["0-ads", "1-speech", "2-music", "9-unsure", "mrs", "todo"]
@ -58,7 +57,12 @@ var getFiles = function(path, after, before, cb) {
var ps = path.split("/");
list[path1] = { class: ps[ps.length-1] };
}
/*if (spf[1].slice(-5) == ".part") {
list[path1]["partial"] = true;
list[path1][spf[1].slice(spf[1].length-5)] = true;
} else {*/
list[path1][spf[1]] = true;
//}
}
}
cb(list);
@ -85,9 +89,10 @@ var findDataFiles = function(options, callback) {
getDirs(options.path + "/records", function(dateDirs) {
for (let i=dateDirs.length-1; i>=0; i--) {
if (timeFrame.after && dateDirs[i] < timeFrame.after) dateDirs.splice(i, 1);
if (timeFrame.before && dateDirs[i] > timeFrame.before) dateDirs.splice(i, 1);
if (timeFrame.after && dateDirs[i] < timeFrame.after.slice(0,10)) dateDirs.splice(i, 1);
if (timeFrame.before && dateDirs[i] > timeFrame.before.slice(0,10)) dateDirs.splice(i, 1);
}
log.debug("findDataFiles: dateDirs:" + JSON.stringify(dateDirs));
async.forEachOf(dateDirs, function(dateDir, index, dateDirCallback) {
getDirs(options.path + "/records/" + dateDir, function(radioDirs) {
@ -95,7 +100,7 @@ var findDataFiles = function(options, callback) {
for (let i=radioDirs.length-1; i>=0; i--) {
if (targetRadios.indexOf(radioDirs[i]) < 0) radioDirs.splice(i, 1);
}
//log.debug("findDataFiles: radioDirs after = " + radioDirs);
log.debug("findDataFiles: radioDirs=" + radioDirs);
async.forEachOf(radioDirs, function(radioDir, index, radioDirCallback) {
async.forEachOf(consts.WLARRAY, function(dataDir, index, dataDirCallback) {
@ -103,7 +108,7 @@ var findDataFiles = function(options, callback) {
fs.stat(path, function(err, stat) {
if (stat && stat.isDirectory()) {
getFiles(path, timeFrame.after, timeFrame.before, function(partialFiles) {
//log.debug("findDataFiles: " + dataDir + ":" + partialFiles);
log.debug("findDataFiles: " + dataDir + ": " + Object.keys(partialFiles).length + " files found");
Object.assign(files[dataDir], partialFiles); // = files[dataDir].concat(fullPathFiles);
dataDirCallback();
});

5
predictor-ml/ml.js
View File

@ -1,7 +1,6 @@
var { Transform } = require("stream");
var cp = require("child_process");
var log = require("loglevel");
log.setLevel("debug");
var log = require("../log.js")("pred-ml");
var consts = {
WLARRAY: ["0-ads", "1-speech", "2-music", "9-unsure", "todo"]
@ -12,7 +11,7 @@ class MlPredictor extends Transform {
super({ readableObjectMode: true });
this.canonical = options.country + "_" + options.name;
var self = this;
// spawn python subprocess
this.cork();
this.predictChild = cp.spawn('python', ['-u', 'mlpredict.py', this.canonical, 11025, 1, 16, 2], { stdio: ['pipe', 'pipe', 'pipe'], cwd: __dirname });