From b8d2b1902a0d92795ccb3a8a97423652877a04ff Mon Sep 17 00:00:00 2001 From: Alexandre Storelli Date: Tue, 10 Apr 2018 11:57:52 +0200 Subject: [PATCH] new logging system --- .gitignore | 5 + backend.js | 25 +- log/.gitkeep | 0 package-lock.json | 519 ++++------------------------------ package.json | 6 +- predictor-db/db.js | 3 +- predictor-db/findDataFiles.js | 17 +- predictor-ml/ml.js | 5 +- 8 files changed, 84 insertions(+), 496 deletions(-) create mode 100644 log/.gitkeep diff --git a/.gitignore b/.gitignore index 85d584d..1ca95fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,7 @@ records/ node_modules/ +log.js +log/ +*.mp3 +*.log +*.png diff --git a/backend.js b/backend.js index 7e850c1..43a671f 100644 --- a/backend.js +++ b/backend.js @@ -1,9 +1,9 @@ var Db = require("./predictor-db/db.js"); var MlPredictor = require("./predictor-ml/ml.js"); -var Dl = require("../adblockradio-dl/dl.js"); +const Codegen = require("stream-audio-fingerprint"); +var { StreamDl } = require("../adblockradio-dl/dl.js"); var { getMeta } = require("webradio-metadata"); -var log = require("loglevel"); -log.setLevel("debug"); +var { log } = require("./log.js")("pred-master"); var country = "France"; var name = "RTL"; @@ -14,7 +14,7 @@ const ENABLE_PREDICTOR_ML = false; const SAVE_AUDIO = true; const FETCH_METADATA = false; -var dl = new Dl({ country: country, name: name, segDuration: 10 }); +var dl = new StreamDl({ country: country, name: name, segDuration: 10 }); dl.on("error", function(err) { console.log("dl err=" + err); }); @@ -31,12 +31,11 @@ var decoder = require('child_process').spawn('ffmpeg', [ //dl.pipe(decoder.stdin); //.write(data); if (ENABLE_PREDICTOR_FINGERPRINT) { - var Codegen = require("stream-audio-fingerprint"); var fingerprinter = new Codegen(); decoder.stdout.pipe(fingerprinter); } if (ENABLE_PREDICTOR_ML) { - var mlPredictor = new MlPredictor({ country: country, name: name}); + var mlPredictor = new MlPredictor({ country: country, name: name }); decoder.stdout.pipe(mlPredictor); } @@ -101,16 +100,10 @@ dl.on("metadata", function(metadata) { if (ENABLE_PREDICTOR_ML) mlPredictor.pipe(dbs.metadata); if (FETCH_METADATA) { getMeta(country, name, function(err, parsedMeta, corsEnabled) { - if (err) { - log.warn("getMeta: error fetching title meta. err=" + err); - } else { - log.info(country + "_" + name + " meta=" + JSON.stringify(parsedMeta)); - if (!dbs.metadata.ended) { - dbs.metadata.write({ type: "title", data: parsedMeta }); - } else { - log.warn("getMeta: could not write metadata, stream already ended"); - } - } + if (err) return log.warn("getMeta: error fetching title meta. err=" + err); + if (dbs.metadata.ended) return log.warn("getMeta: could not write metadata, stream already ended"); + log.info(country + "_" + name + " meta=" + JSON.stringify(parsedMeta)); + dbs.metadata.write({ type: "title", data: parsedMeta }); }); } decoder.stdin.write(dataObj.data); diff --git a/log/.gitkeep b/log/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/package-lock.json b/package-lock.json index 940ce48..8903677 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,20 +4,14 @@ "lockfileVersion": 1, "requires": true, "dependencies": { - "accepts": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.4.tgz", - "integrity": "sha1-hiRnWMfdbSGmR0/whKR0DsBesh8=", + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", "requires": { - "mime-types": "2.1.17", - "negotiator": "0.6.1" + "color-convert": "1.9.1" } }, - "array-flatten": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", - "integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI=" - }, "async": { "version": "2.6.0", "resolved": "https://registry.npmjs.org/async/-/async-2.6.0.tgz", @@ -26,296 +20,61 @@ "lodash": "4.17.4" } }, - "body-parser": { - "version": "1.18.2", - "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz", - "integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=", + "chalk": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.3.2.tgz", + "integrity": "sha512-ZM4j2/ld/YZDc3Ma8PgN7gyAk+kHMMMyzLNryCPGhWrsfAuDVeuid5bpRFTDgMH9JBK2lA4dyyAkkZYF/WcqDQ==", "requires": { - "bytes": "3.0.0", - "content-type": "1.0.4", - "debug": "2.6.9", - "depd": "1.1.1", - "http-errors": "1.6.2", - "iconv-lite": "0.4.19", - "on-finished": "2.3.0", - "qs": "6.5.1", - "raw-body": "2.3.2", - "type-is": "1.6.15" + "ansi-styles": "3.2.1", + "escape-string-regexp": "1.0.5", + "supports-color": "5.3.0" } }, - "bytes": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", - "integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg=" - }, - "camelize": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/camelize/-/camelize-1.0.0.tgz", - "integrity": "sha1-FkpUg+Yw+kMh5a8HAg5TGDGyYJs=" - }, "chunked-stream": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/chunked-stream/-/chunked-stream-0.0.2.tgz", "integrity": "sha1-zPdQmNrb8Ls4/zB+EJWCeAUHk54=" }, - "content-disposition": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz", - "integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ=" - }, - "content-security-policy-builder": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/content-security-policy-builder/-/content-security-policy-builder-1.1.0.tgz", - "integrity": "sha1-2R8bB2I2wRmFDH3umSS/VeBXcrM=", + "color-convert": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.1.tgz", + "integrity": "sha512-mjGanIiwQJskCC18rPR6OmrZ6fm2Lc7PeGFYwCmy5J34wC6F1PzdGL6xeMfmgicfYcNLGuVFA3WzXtIDCQSZxQ==", "requires": { - "dashify": "0.2.2" + "color-name": "1.1.3" } }, - "content-type": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz", - "integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA==" - }, - "cookie": { - "version": "0.3.1", - "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz", - "integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s=" - }, - "cookie-signature": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", - "integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw=" - }, - "dasherize": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/dasherize/-/dasherize-2.0.0.tgz", - "integrity": "sha1-bYCcnNDPe7iVLYD8hPoT1H3bEwg=" - }, - "dashify": { - "version": "0.2.2", - "resolved": "https://registry.npmjs.org/dashify/-/dashify-0.2.2.tgz", - "integrity": "sha1-agdBWgHJH69KMuONnfunH2HLIP4=" - }, - "debug": { - "version": "2.6.9", - "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", - "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", - "requires": { - "ms": "2.0.0" - } - }, - "depd": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz", - "integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k=" - }, - "destroy": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz", - "integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA=" - }, - "dns-prefetch-control": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/dns-prefetch-control/-/dns-prefetch-control-0.1.0.tgz", - "integrity": "sha1-YN20V3dOF48flBXwyrsOhbCzALI=" - }, - "dont-sniff-mimetype": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/dont-sniff-mimetype/-/dont-sniff-mimetype-1.0.0.tgz", - "integrity": "sha1-WTKJDcn04vGeXrAqIAJuXl78j1g=" + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" }, "dsp.js": { - "version": "git+https://git@github.com/corbanbrook/dsp.js.git#52d60e4ece9158b2dac9261790d45421a1556cae" + "version": "git+https://git@github.com/corbanbrook/dsp.js.git#051f341d8e1d05515efecb1851bbc862decf02bc" }, - "ee-first": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", - "integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0=" + "escape-string-regexp": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", + "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=" }, - "encodeurl": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.1.tgz", - "integrity": "sha1-eePVhlU0aQn+bw9Fpd5oEDspTSA=" - }, - "escape-html": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", - "integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg=" - }, - "etag": { - "version": "1.8.1", - "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", - "integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc=" - }, - "expect-ct": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/expect-ct/-/expect-ct-0.1.0.tgz", - "integrity": "sha1-UnNWeN4YUwiQ2Ne5XwrGNkCVgJQ=" - }, - "express": { - "version": "4.16.2", - "resolved": "https://registry.npmjs.org/express/-/express-4.16.2.tgz", - "integrity": "sha1-41xt/i1kt9ygpc1PIXgb4ymeB2w=", - "requires": { - "accepts": "1.3.4", - "array-flatten": "1.1.1", - "body-parser": "1.18.2", - "content-disposition": "0.5.2", - "content-type": "1.0.4", - "cookie": "0.3.1", - "cookie-signature": "1.0.6", - "debug": "2.6.9", - "depd": "1.1.1", - "encodeurl": "1.0.1", - "escape-html": "1.0.3", - "etag": "1.8.1", - "finalhandler": "1.1.0", - "fresh": "0.5.2", - "merge-descriptors": "1.0.1", - "methods": "1.1.2", - "on-finished": "2.3.0", - "parseurl": "1.3.2", - "path-to-regexp": "0.1.7", - "proxy-addr": "2.0.2", - "qs": "6.5.1", - "range-parser": "1.2.0", - "safe-buffer": "5.1.1", - "send": "0.16.1", - "serve-static": "1.13.1", - "setprototypeof": "1.1.0", - "statuses": "1.3.1", - "type-is": "1.6.15", - "utils-merge": "1.0.1", - "vary": "1.1.2" - } - }, - "finalhandler": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.0.tgz", - "integrity": "sha1-zgtoVbRYU+eRsvzGgARtiCU91/U=", - "requires": { - "debug": "2.6.9", - "encodeurl": "1.0.1", - "escape-html": "1.0.3", - "on-finished": "2.3.0", - "parseurl": "1.3.2", - "statuses": "1.3.1", - "unpipe": "1.0.0" - } - }, - "forwarded": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz", - "integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ=" - }, - "frameguard": { + "has-flag": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/frameguard/-/frameguard-3.0.0.tgz", - "integrity": "sha1-e8rUae57lukdEs6zlZx4I1qScuk=" - }, - "fresh": { - "version": "0.5.2", - "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", - "integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac=" - }, - "helmet": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/helmet/-/helmet-3.9.0.tgz", - "integrity": "sha512-czCyS77TyanWlfVSoGlb9GBJV2Q2zJayKxU5uBw0N1TzDTs/qVNh1SL8Q688KU0i0Sb7lQ/oLtnaEqXzl2yWvA==", - "requires": { - "dns-prefetch-control": "0.1.0", - "dont-sniff-mimetype": "1.0.0", - "expect-ct": "0.1.0", - "frameguard": "3.0.0", - "helmet-csp": "2.6.0", - "hide-powered-by": "1.0.0", - "hpkp": "2.0.0", - "hsts": "2.1.0", - "ienoopen": "1.0.0", - "nocache": "2.0.0", - "referrer-policy": "1.1.0", - "x-xss-protection": "1.0.0" - } - }, - "helmet-csp": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/helmet-csp/-/helmet-csp-2.6.0.tgz", - "integrity": "sha512-n/oW9l6RtO4f9YvphsNzdvk1zITrSN7iRT8ojgrJu/N3mVdHl9zE4OjbiHWcR64JK32kbqx90/yshWGXcjUEhw==", - "requires": { - "camelize": "1.0.0", - "content-security-policy-builder": "1.1.0", - "dasherize": "2.0.0", - "lodash.reduce": "4.6.0", - "platform": "1.3.4" - } - }, - "hide-powered-by": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/hide-powered-by/-/hide-powered-by-1.0.0.tgz", - "integrity": "sha1-SoWtZYgfYoV/xwr3F0oRhNzM4ys=" - }, - "hpkp": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/hpkp/-/hpkp-2.0.0.tgz", - "integrity": "sha1-EOFCJk52IVpdMMROxD3mTe5tFnI=" - }, - "hsts": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/hsts/-/hsts-2.1.0.tgz", - "integrity": "sha512-zXhh/DqgrTXJ7erTN6Fh5k/xjMhDGXCqdYN3wvxUvGUQvnxcFfUd8E+6vLg/nk3ss1TYMb+DhRl25fYABioTvA==" - }, - "http-errors": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz", - "integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=", - "requires": { - "depd": "1.1.1", - "inherits": "2.0.3", - "setprototypeof": "1.0.3", - "statuses": "1.3.1" - }, - "dependencies": { - "setprototypeof": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz", - "integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ=" - } - } - }, - "iconv-lite": { - "version": "0.4.19", - "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz", - "integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ==" - }, - "ienoopen": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/ienoopen/-/ienoopen-1.0.0.tgz", - "integrity": "sha1-NGpCj0dKrI9QzzeE6i0PFvYr2ms=" - }, - "inherits": { - "version": "2.0.3", - "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", - "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=" - }, - "ipaddr.js": { - "version": "1.5.2", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.5.2.tgz", - "integrity": "sha1-1LUFvemUaYfM8PxY2QEP+WB+P6A=" + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=" }, "lodash": { "version": "4.17.4", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.4.tgz", "integrity": "sha1-eCA6TRwyiuHYbcpkYONptX9AVa4=" }, - "lodash.reduce": { - "version": "4.6.0", - "resolved": "https://registry.npmjs.org/lodash.reduce/-/lodash.reduce-4.6.0.tgz", - "integrity": "sha1-8atrg5KZrUj3hKu/R2WW8DuRTTs=" - }, "loglevel": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.0.tgz", - "integrity": "sha1-rgyqVhERSYxboTcj1vtjHSQAOTQ=" + "version": "1.6.1", + "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.6.1.tgz", + "integrity": "sha1-4PyVEztu8nbNyIh82vJKpvFW+Po=" + }, + "loglevel-plugin-prefix": { + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/loglevel-plugin-prefix/-/loglevel-plugin-prefix-0.8.3.tgz", + "integrity": "sha512-oFDEE3krjFTlyXfdT6shN4HsIlDrbPyleI2WIgRfn//oUEVfe8dP7goO9ktTSdOQC08CTKshKHHdZXe4Dy9TxQ==" }, "m3u8": { "version": "0.0.7", @@ -325,168 +84,21 @@ "chunked-stream": "0.0.2" } }, - "media-typer": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", - "integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=" - }, - "merge-descriptors": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz", - "integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E=" - }, - "methods": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", - "integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=" - }, - "mime": { - "version": "1.4.1", - "resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz", - "integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ==" - }, - "mime-db": { - "version": "1.30.0", - "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.30.0.tgz", - "integrity": "sha1-dMZD2i3Z1qRTmZY0ZbJtXKfXHwE=" - }, - "mime-types": { - "version": "2.1.17", - "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.17.tgz", - "integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=", - "requires": { - "mime-db": "1.30.0" - } - }, - "ms": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", - "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" - }, "nan": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.7.0.tgz", "integrity": "sha1-2Vv3IeyHfgjbJ27T/G63j5CDrUY=" }, - "negotiator": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz", - "integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk=" - }, - "nocache": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/nocache/-/nocache-2.0.0.tgz", - "integrity": "sha1-ICtIAhoMTL3i34DeFaF0Q8i0OYA=" - }, "node-png": { "version": "0.4.3", "resolved": "https://registry.npmjs.org/node-png/-/node-png-0.4.3.tgz", "integrity": "sha1-RQIjeWuC08yg/+Sl1cf6l0hZdOc=" }, - "on-finished": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", - "integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=", - "requires": { - "ee-first": "1.1.1" - } - }, - "parseurl": { - "version": "1.3.2", - "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz", - "integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M=" - }, - "path-to-regexp": { - "version": "0.1.7", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", - "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=" - }, - "platform": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/platform/-/platform-1.3.4.tgz", - "integrity": "sha1-bw+xftqqSPIUQrOpdcBjEw8cPr0=" - }, - "proxy-addr": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.2.tgz", - "integrity": "sha1-ZXFQT0e7mI7IGAJT+F3X4UlSvew=", - "requires": { - "forwarded": "0.1.2", - "ipaddr.js": "1.5.2" - } - }, - "qs": { - "version": "6.5.1", - "resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz", - "integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A==" - }, - "range-parser": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz", - "integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4=" - }, - "raw-body": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz", - "integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=", - "requires": { - "bytes": "3.0.0", - "http-errors": "1.6.2", - "iconv-lite": "0.4.19", - "unpipe": "1.0.0" - } - }, - "referrer-policy": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/referrer-policy/-/referrer-policy-1.1.0.tgz", - "integrity": "sha1-NXdOtzW/UPtsB46DM0tHI1AgfXk=" - }, - "safe-buffer": { - "version": "5.1.1", - "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", - "integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg==" - }, "sax": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz", "integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw==" }, - "send": { - "version": "0.16.1", - "resolved": "https://registry.npmjs.org/send/-/send-0.16.1.tgz", - "integrity": "sha512-ElCLJdJIKPk6ux/Hocwhk7NFHpI3pVm/IZOYWqUmoxcgeyM+MpxHHKhb8QmlJDX1pU6WrgaHBkVNm73Sv7uc2A==", - "requires": { - "debug": "2.6.9", - "depd": "1.1.1", - "destroy": "1.0.4", - "encodeurl": "1.0.1", - "escape-html": "1.0.3", - "etag": "1.8.1", - "fresh": "0.5.2", - "http-errors": "1.6.2", - "mime": "1.4.1", - "ms": "2.0.0", - "on-finished": "2.3.0", - "range-parser": "1.2.0", - "statuses": "1.3.1" - } - }, - "serve-static": { - "version": "1.13.1", - "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.1.tgz", - "integrity": "sha512-hSMUZrsPa/I09VYFJwa627JJkNs0NrfL1Uzuup+GqHfToR2KcsXFymXSV90hoyw3M+msjFuQly+YzIH/q0MGlQ==", - "requires": { - "encodeurl": "1.0.1", - "escape-html": "1.0.3", - "parseurl": "1.3.2", - "send": "0.16.1" - } - }, - "setprototypeof": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz", - "integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ==" - }, "sqlite3": { "version": "3.1.13", "resolved": "https://registry.npmjs.org/sqlite3/-/sqlite3-3.1.13.tgz", @@ -1199,72 +811,47 @@ } } }, - "statuses": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz", - "integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4=" - }, "stream-audio-fingerprint": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/stream-audio-fingerprint/-/stream-audio-fingerprint-1.0.1.tgz", "integrity": "sha512-16YwNlkjLVr7SfO+vZBisJ0cQ5XAy0tVN6HAlv5N6FpY5HPMHo0cM3L/B67NAevNCTjqUH8JbNQeSy2aaeugaQ==", "requires": { - "dsp.js": "git+https://git@github.com/corbanbrook/dsp.js.git#52d60e4ece9158b2dac9261790d45421a1556cae", + "dsp.js": "git+https://git@github.com/corbanbrook/dsp.js.git#051f341d8e1d05515efecb1851bbc862decf02bc", "node-png": "0.4.3" } }, - "type-is": { - "version": "1.6.15", - "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.15.tgz", - "integrity": "sha1-yrEPtJCeRByChC6v4a1kbIGARBA=", + "supports-color": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.3.0.tgz", + "integrity": "sha512-0aP01LLIskjKs3lq52EC0aGBAJhLq7B2Rd8HC/DR/PtNNpcLilNmHC12O+hu0usQpo7wtHNRqtrhBwtDb0+dNg==", "requires": { - "media-typer": "0.3.0", - "mime-types": "2.1.17" + "has-flag": "3.0.0" } }, - "unpipe": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", - "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" - }, - "utils-merge": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", - "integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM=" - }, - "vary": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", - "integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=" - }, "webradio-metadata": { - "version": "0.1.3", - "resolved": "https://registry.npmjs.org/webradio-metadata/-/webradio-metadata-0.1.3.tgz", - "integrity": "sha512-X+9q4IdO7Kbf4JN+V/+RSnO0XaBW+VAxGssyYlni4j9AnfpQ4ilWJiqeWdrkjXHXRikATDbvKBgYHlHZBd86sg==", + "version": "0.1.15", + "resolved": "https://registry.npmjs.org/webradio-metadata/-/webradio-metadata-0.1.15.tgz", + "integrity": "sha512-Tuue71q1aEZg2Bls0LoWi3SlSMRJuF+uF7Z/e7FducahMWeIbKSNyl8mZjY76D9KQCOl9XdXxvmcbMO6Zt2DDA==", "requires": { - "express": "4.16.2", - "helmet": "3.9.0", + "chalk": "2.3.2", + "loglevel": "1.6.1", + "loglevel-plugin-prefix": "0.8.3", "xml2js": "0.4.19" } }, - "x-xss-protection": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/x-xss-protection/-/x-xss-protection-1.0.0.tgz", - "integrity": "sha1-iYr7k4abJGYc+cUvnujbjtB2Tdk=" - }, "xml2js": { "version": "0.4.19", "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", "requires": { "sax": "1.2.4", - "xmlbuilder": "9.0.4" + "xmlbuilder": "9.0.7" } }, "xmlbuilder": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.4.tgz", - "integrity": "sha1-UZy0ymhtAFqEINNJbz8MruzKWA8=" + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" } } } diff --git a/package.json b/package.json index 6feb645..5ff5d30 100644 --- a/package.json +++ b/package.json @@ -17,11 +17,11 @@ "dependencies": { "async": "^2.6.0", "dsp.js": "git+https://git@github.com/corbanbrook/dsp.js.git", - "loglevel": "^1.6.0", + "loglevel": "^1.6.1", "m3u8": "0.0.7", "node-png": "^0.4.3", "sqlite3": "^3.1.13", - "stream-audio-fingerprint": "^1.0.1", - "webradio-metadata": "0.1.3" + "stream-audio-fingerprint": "1.0.x", + "webradio-metadata": "^0.1.15" } } diff --git a/predictor-db/db.js b/predictor-db/db.js index 6105b32..e919f59 100644 --- a/predictor-db/db.js +++ b/predictor-db/db.js @@ -1,8 +1,7 @@ var sqlite3 = require("sqlite3").verbose(); var { Writable, Transform } = require("stream"); var fs = require("fs"); -var log = require("loglevel"); -log.setLevel("debug"); +var { log } = require("../log.js")("pred-db"); var cp = require("child_process"); var async = require("async"); var findDataFiles = require("./findDataFiles.js"); diff --git a/predictor-db/findDataFiles.js b/predictor-db/findDataFiles.js index 2eb3009..2b24d31 100644 --- a/predictor-db/findDataFiles.js +++ b/predictor-db/findDataFiles.js @@ -1,6 +1,5 @@ var fs = require("fs"); -var log = require("loglevel"); -log.setLevel("debug"); +const { log } = require("../log.js")("pred-db-findDataFiles"); var async = require("async"); var consts = { WLARRAY: ["0-ads", "1-speech", "2-music", "9-unsure", "mrs", "todo"] @@ -58,7 +57,12 @@ var getFiles = function(path, after, before, cb) { var ps = path.split("/"); list[path1] = { class: ps[ps.length-1] }; } + /*if (spf[1].slice(-5) == ".part") { + list[path1]["partial"] = true; + list[path1][spf[1].slice(spf[1].length-5)] = true; + } else {*/ list[path1][spf[1]] = true; + //} } } cb(list); @@ -85,9 +89,10 @@ var findDataFiles = function(options, callback) { getDirs(options.path + "/records", function(dateDirs) { for (let i=dateDirs.length-1; i>=0; i--) { - if (timeFrame.after && dateDirs[i] < timeFrame.after) dateDirs.splice(i, 1); - if (timeFrame.before && dateDirs[i] > timeFrame.before) dateDirs.splice(i, 1); + if (timeFrame.after && dateDirs[i] < timeFrame.after.slice(0,10)) dateDirs.splice(i, 1); + if (timeFrame.before && dateDirs[i] > timeFrame.before.slice(0,10)) dateDirs.splice(i, 1); } + log.debug("findDataFiles: dateDirs:" + JSON.stringify(dateDirs)); async.forEachOf(dateDirs, function(dateDir, index, dateDirCallback) { getDirs(options.path + "/records/" + dateDir, function(radioDirs) { @@ -95,7 +100,7 @@ var findDataFiles = function(options, callback) { for (let i=radioDirs.length-1; i>=0; i--) { if (targetRadios.indexOf(radioDirs[i]) < 0) radioDirs.splice(i, 1); } - //log.debug("findDataFiles: radioDirs after = " + radioDirs); + log.debug("findDataFiles: radioDirs=" + radioDirs); async.forEachOf(radioDirs, function(radioDir, index, radioDirCallback) { async.forEachOf(consts.WLARRAY, function(dataDir, index, dataDirCallback) { @@ -103,7 +108,7 @@ var findDataFiles = function(options, callback) { fs.stat(path, function(err, stat) { if (stat && stat.isDirectory()) { getFiles(path, timeFrame.after, timeFrame.before, function(partialFiles) { - //log.debug("findDataFiles: " + dataDir + ":" + partialFiles); + log.debug("findDataFiles: " + dataDir + ": " + Object.keys(partialFiles).length + " files found"); Object.assign(files[dataDir], partialFiles); // = files[dataDir].concat(fullPathFiles); dataDirCallback(); }); diff --git a/predictor-ml/ml.js b/predictor-ml/ml.js index eb2fe93..d7e2bac 100644 --- a/predictor-ml/ml.js +++ b/predictor-ml/ml.js @@ -1,7 +1,6 @@ var { Transform } = require("stream"); var cp = require("child_process"); -var log = require("loglevel"); -log.setLevel("debug"); +var log = require("../log.js")("pred-ml"); var consts = { WLARRAY: ["0-ads", "1-speech", "2-music", "9-unsure", "todo"] @@ -12,7 +11,7 @@ class MlPredictor extends Transform { super({ readableObjectMode: true }); this.canonical = options.country + "_" + options.name; var self = this; - + // spawn python subprocess this.cork(); this.predictChild = cp.spawn('python', ['-u', 'mlpredict.py', this.canonical, 11025, 1, 16, 2], { stdio: ['pipe', 'pipe', 'pipe'], cwd: __dirname });