diff --git a/check-updates.js b/check-updates.js index 9bfff87..daf7f5c 100644 --- a/check-updates.js +++ b/check-updates.js @@ -4,67 +4,87 @@ const axios = require('axios'); const tar = require('tar'); const fs = require('fs-extra'); const { log } = require('abr-log')('checkModelUpdates'); +const assert = require('assert'); const MODELS_REPOSITORY = 'https://www.adblockradio.com/models/'; const METADATA_REPOSITORY = 'https://www.adblockradio.com/metadata/'; const CHECKSUM_SUFFIX = '.sha256sum'; -const isToUpdate = async function(localPath, remotePath, file) { +const isToUpdate = async function(localFile, remoteFile) { let localChecksum = null; try { - localChecksum = await fs.readFile(localPath + '/' + file + CHECKSUM_SUFFIX); + localChecksum = await fs.readFile(localFile + CHECKSUM_SUFFIX); } catch (e) { - log.info('checksum for ' + localPath + '/' + file + ' not found. Will fetch models.'); + log.info('checksum for ' + localFile + ' not found. Will fetch models.'); return true; } - const remoteFile = remotePath + file + CHECKSUM_SUFFIX; try { - const remoteChecksum = await axios.get(encodeURI(remoteFile)); + const remoteChecksum = await axios.get(encodeURI(remoteFile + CHECKSUM_SUFFIX)); if ('' + localChecksum !== '' + remoteChecksum.data) { //log.info('different checksums local=' + localChecksum + ' remote=' + remoteChecksum.data); return true; } else { - log.info(file + ' is up to date'); + log.info(localFile + ' is up to date'); return false; } } catch (e) { - log.warn('could not fetch ' + remoteFile + '. err=' + e); + log.warn('could not fetch ' + remoteFile + CHECKSUM_SUFFIX + '. err=' + e); return false; } } -const update = async function(localPath, remotePath, file) { - log.info('update ' + localPath + '/' + file); +const update = async function(remoteFile, localFile, options) { //localPath, remotePath, file) { + log.info('update ' + localFile); try { - const checksumData = await axios.get(encodeURI(remotePath + file + CHECKSUM_SUFFIX)); - await fs.writeFile(localPath + '/' + file + CHECKSUM_SUFFIX, checksumData.data); - const data = await axios.get(encodeURI(remotePath + file), { responseType: 'arraybuffer' }); - await fs.writeFile(localPath + '/' + file, data.data); - await tar.x({ file: localPath + '/' + file, cwd: localPath, strict: true }); - await fs.unlink(localPath + '/' + file); + const localFileSplit = localFile.split('/'); + const localPath = localFileSplit.slice(0, localFileSplit.length - 1).join('/'); + //log.debug("localPath=" + localPath); + try { + await fs.mkdir(localPath, { recursive: true }); + } catch (e) { + if (!('' + e).includes('EEXIST')) { + log.error("Cannot create model directory " + localPath); + throw e; + } + } + const checksumData = await axios.get(encodeURI(remoteFile + CHECKSUM_SUFFIX)); + await fs.writeFile(localFile + CHECKSUM_SUFFIX, checksumData.data); + const data = await axios.get(encodeURI(remoteFile), { responseType: 'arraybuffer' }); + await fs.writeFile(localFile, data.data); + if (options && options.untar) { + await tar.x({ file: localFile, cwd: localPath, strict: true }); + await fs.unlink(localFile); + } } catch (e) { - log.warn('could not update with remote ' + remotePath + file + '. err=' + e); + log.warn('could not update with remote ' + remoteFile + '. err=' + e); } } -exports.checkModelUpdates = async function(country, name, modelsPath, mlUpdateCallback, hotlistUpdateCallback) { - const canonical = country + '_' + name; - const modelFile = canonical + '.keras.tar.gz'; - if (await isToUpdate(modelsPath, MODELS_REPOSITORY, modelFile)) { - await update(modelsPath, MODELS_REPOSITORY, modelFile); - if (mlUpdateCallback) mlUpdateCallback(); - } - const hotlistFile = canonical + '.sqlite.tar.gz'; - if (await isToUpdate(modelsPath, MODELS_REPOSITORY, hotlistFile)) { - await update(modelsPath, MODELS_REPOSITORY, hotlistFile); - if (hotlistUpdateCallback) hotlistUpdateCallback(); +exports.checkModelUpdates = async function(params) { + assert(params.localPath); + assert(params.files); + + //const canonical = params.country + '_' + params.name; + + for (let i=0; i= 0) { log.warn(self.canonical + " lost remote Python worker. will restart it"); self.mlPredictor.destroy();