<commit_before>/*    Copyright 2021 Firewalla INC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

'use strict';

const fs = require('fs');
const Promise = require('bluebird');
Promise.promisifyAll(fs);

const jsonfile = require('jsonfile');
const jsonReadFileAsync = Promise.promisify(jsonfile.readFile);
const jsonWriteFileAsync = Promise.promisify(jsonfile.writeFile);

const f = require('../../net2/Firewalla.js');
const cacheFolder = `${f.getRuntimeInfoFolder()}/cache`;
const log = require('../../net2/logger.js')(__filename);
const bone = require("../../lib/Bone.js");
const sclient = require('../../util/redis_manager.js').getSubscriptionClient();
const config = require('../../net2/config.js').getConfig();

const expirationDays = (config.cloudcache && config.cloudcache.expirationDays) || 30;
const _ = require('lodash');

let instance = null;

class CloudCacheItem {
  constructor(name) {
    this.localCachePath = `${cacheFolder}/${name}`;
    this.localMetadataPath = `${this.localCachePath}.metadata`;
    this.cloudHashKey = name;
    this.cloudMetadataHashKey = `metadata:${name}`;
    this.name = name;
  }

  async getLocalCacheContent() {
    try {
      const content = await fs.readFileAsync(this.localCachePath, 'utf8');
      return content;
    } catch (e) {
      return null;
    }
  }

  async getLocalMetadata() {
    try {
      const data = await jsonReadFileAsync(this.localMetadataPath);
      return data;
    } catch (err) {
      log.debug("Failed to load local matadata:", this.localMetadataPath);
      return null;
    }
  }

  async writeLocalCacheContent(data) {
    return fs.writeFileAsync(this.localCachePath, data);
  }

  async writeLocalMetadata(metadata) {
    return jsonWriteFileAsync(this.localMetadataPath, metadata);
  }

  async getCloudMetadata() {
    try {
      const jsonString = await bone.hashsetAsync(this.cloudMetadataHashKey);
      return JSON.parse(jsonString);
    } catch (err) {
      log.error("Failed to load cloud metadata, err:", err);
      return null;
    }
  }

  async getCloudData() {
    return bone.hashsetAsync(this.cloudHashKey);
  }

  isExpired(currentTime, lastUpdateTime) {
    if (!currentTime || !lastUpdateTime) {
      return false;
    }
    const ageInDays = (currentTime - lastUpdateTime) / 86400;
    if (ageInDays > expirationDays) {
      return true;
    }
    return false;
  }

  async cleanUp() {
    try {
      await fs.unlinkAsync(this.localCachePath);
      await fs.unlinkAsync(this.localMetadataPath);
    } catch (e) {
      //
    }
  }

  async download(alwaysOnUpdate = false) {
    // return true if cache is supported on this key
    // return false if cache is not supported on this key

    let localMetadata = await this.getLocalMetadata();
    const cloudMetadata = await this.getCloudMetadata();

    // return false if cloud cache is not supported for this key.
    if (!localMetadata && (!cloudMetadata || _.isEmpty(cloudMetadata))) {
      if (this.onUpdateCallback) {
        this.onUpdateCallback(null);
      }
      return false;
    }

    const currentTime = new Date().getTime() / 1000;
    let needDownload = true;


    // cloud metadata doesn't exist.
    if (localMetadata && _.isEmpty(cloudMetadata) || !cloudMetadata.updated || !cloudMetadata.sha256sum) {
      log.info(`Invalid file ${this.name} from cloud, ignored`);
      needDownload = false;
    }

    // protect from server time screw
    if (cloudMetadata && currentTime < cloudMetadata.updated) {
      cloudMetadata.updated = currentTime;
    }

    // cloud metadata has same checksum as local one. update timestamp
    if (localMetadata && cloudMetadata &&
      localMetadata.sha256sum && cloudMetadata.sha256sum &&
      localMetadata.sha256sum === cloudMetadata.sha256sum) {
      if (localMetadata.updated < cloudMetadata.updated) {
        await this.writeLocalMetadata(cloudMetadata);
      }
      log.info(`skip updating, cache ${this.name} is already up to date`);
      needDownload = false;
    }

    // cloud metadata has different checksum but with older timestamp than current one. Just ignore remote one. This is unlikely to occur.
    if (localMetadata && cloudMetadata && localMetadata.sha256sum && cloudMetadata.sha256sum && localMetadata.sha256sum !== cloudMetadata.sha256sum && cloudMetadata.updated < localMetadata.updated) {
      log.info(`cloud metadata for ${this.name} is older than local one. skip updating`);
      needDownload = false;
    }

    let localContent = null;
    let hasNewData = false;

    // download cloud data if needed.
    if (needDownload && !this.isExpired(currentTime, cloudMetadata.updated)) {
      log.info(`Downloading ${this.cloudHashKey}...`);
      const cloudContent = await this.getCloudData();
      log.info(`Download Complete for ${this.cloudHashKey}!`);
      await this.writeLocalCacheContent(cloudContent);
      await this.writeLocalMetadata(cloudMetadata);
      localContent = cloudContent;
      localMetadata = cloudMetadata;

      let updatedTime = "unknown";
      if (cloudMetadata.updated) {
        updatedTime = new Date(cloudMetadata.updated * 1000);
      }
      hasNewData = true;
      log.info(`Updating cache file ${this.name}, updated at ${updatedTime}`);
    }

    // check local data and update
    if (!localContent) {
      localContent = await this.getLocalCacheContent();
    }

    if (localMetadata && this.isExpired(currentTime, localMetadata.updated)) {<commit_msg>check cloud cache integrity in cloudcache.js<commit_after>/*    Copyright 2021 Firewalla INC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

'use strict';

const fs = require('fs');
const Promise = require('bluebird');
Promise.promisifyAll(fs);

const jsonfile = require('jsonfile');
const jsonReadFileAsync = Promise.promisify(jsonfile.readFile);
const jsonWriteFileAsync = Promise.promisify(jsonfile.writeFile);

const f = require('../../net2/Firewalla.js');
const cacheFolder = `${f.getRuntimeInfoFolder()}/cache`;
const log = require('../../net2/logger.js')(__filename);
const bone = require("../../lib/Bone.js");
const sclient = require('../../util/redis_manager.js').getSubscriptionClient();
const config = require('../../net2/config.js').getConfig();
const crypto = require('crypto');

const expirationDays = (config.cloudcache && config.cloudcache.expirationDays) || 30;
const _ = require('lodash');

let instance = null;

class CloudCacheItem {
  constructor(name) {
    this.localCachePath = `${cacheFolder}/${name}`;
    this.localMetadataPath = `${this.localCachePath}.metadata`;
    this.cloudHashKey = name;
    this.cloudMetadataHashKey = `metadata:${name}`;
    this.name = name;
  }

  async getLocalCacheContent() {
    try {
      const content = await fs.readFileAsync(this.localCachePath, 'utf8');
      return content;
    } catch (e) {
      return null;
    }
  }

  async getLocalMetadata() {
    try {
      const data = await jsonReadFileAsync(this.localMetadataPath);
      return data;
    } catch (err) {
      log.debug("Failed to load local matadata:", this.localMetadataPath);
      return null;
    }
  }

  async checkLocalCacheIntegrity(expectedSha256) {
    const content = await fs.readFileAsync(this.localCachePath).catch(err => null);
    if (!content)
      return false;
    try {
      const actualSha256 = crypto.createHash('sha256').update(content).digest('hex');
      return actualSha256 == expectedSha256;
    } catch (err) {
      log.error(`Failed to calculate local content hash for ${this.name}`, err.message);
      return false;
    }
  }

  async writeLocalCacheContent(data) {
    return fs.writeFileAsync(this.localCachePath, data);
  }

  async writeLocalMetadata(metadata) {
    return jsonWriteFileAsync(this.localMetadataPath, metadata);
  }

  async getCloudMetadata() {
    try {
      const jsonString = await bone.hashsetAsync(this.cloudMetadataHashKey);
      return JSON.parse(jsonString);
    } catch (err) {
      log.error("Failed to load cloud metadata, err:", err);
      return null;
    }
  }

  async getCloudData() {
    return bone.hashsetAsync(this.cloudHashKey);
  }

  isExpired(currentTime, lastUpdateTime) {
    if (!currentTime || !lastUpdateTime) {
      return false;
    }
    const ageInDays = (currentTime - lastUpdateTime) / 86400;
    if (ageInDays > expirationDays) {
      return true;
    }
    return false;
  }

  async cleanUp() {
    try {
      await fs.unlinkAsync(this.localCachePath);
      await fs.unlinkAsync(this.localMetadataPath);
    } catch (e) {
      //
    }
  }

  async download(alwaysOnUpdate = false) {
    // return true if cache is supported on this key
    // return false if cache is not supported on this key

    let localMetadata = await this.getLocalMetadata();
    const cloudMetadata = await this.getCloudMetadata();

    // return false if cloud cache is not supported for this key.
    if (!localMetadata && (!cloudMetadata || _.isEmpty(cloudMetadata))) {
      if (this.onUpdateCallback) {
        this.onUpdateCallback(null);
      }
      return false;
    }

    const currentTime = new Date().getTime() / 1000;
    let needDownload = true;

    let localIntegrity = false;
    if (localMetadata && localMetadata.sha256sum) {
      localIntegrity = await this.checkLocalCacheIntegrity(localMetadata.sha256sum);
    }

    // cloud metadata doesn't exist.
    if (localMetadata && _.isEmpty(cloudMetadata) || !cloudMetadata.updated || !cloudMetadata.sha256sum) {
      log.info(`Invalid file ${this.name} from cloud, ignored`);
      needDownload = false;
    }

    // protect from server time screw
    if (cloudMetadata && currentTime < cloudMetadata.updated) {
      cloudMetadata.updated = currentTime;
    }

    // cloud metadata has same checksum as local one. update timestamp
    if (localMetadata && cloudMetadata &&
      localMetadata.sha256sum && cloudMetadata.sha256sum &&
      localMetadata.sha256sum === cloudMetadata.sha256sum) {
      if (localMetadata.updated < cloudMetadata.updated) {
        await this.writeLocalMetadata(cloudMetadata);
      }
      if (localIntegrity) {
        log.info(`skip updating, cache ${this.name} is already up to date`);
        needDownload = false;
      } else {
        log.warn(`local cache content sha256 of ${this.name} mismatches with that in local metadata, need to download from cloud`);
      }
    }

    // cloud metadata has different checksum but with older timestamp than current one. Just ignore remote one. This is unlikely to occur.
    if (localMetadata && cloudMetadata && localIntegrity && localMetadata.sha256sum && cloudMetadata.sha256sum && localMetadata.sha256sum !== cloudMetadata.sha256sum && cloudMetadata.updated < localMetadata.updated) {
      log.info(`cloud metadata for ${this.name} is older than local one. skip updating`);
      needDownload = false;
    }

    let localContent = null;
    let hasNewData = false;

    // download cloud data if needed.
    if (needDownload && !this.isExpired(currentTime, cloudMetadata.updated)) {
      log.info(`Downloading ${this.cloudHashKey}...`);
      const cloudContent = await this.getCloudData();
      log.info(`Download Complete for ${this.cloudHashKey}!`);
      await this.writeLocalCacheContent(cloudContent);
      await this.writeLocalMetadata(cloudMetadata);
      localContent = cloudContent;
      localMetadata = cloudMetadata;

      let updatedTime = "unknown";
      if (cloudMetadata.updated) {
        updatedTime = new Date(cloudMetadata.updated * 1000);
      }
      hasNewData = true;
      log.info(`Updating cache file ${this.name}, updated at ${updatedTime}`);
    }

    // check local data and update
    if (!localContent) {
      localContent = await this.getLocalCacheContent();
    }

    if (localMetadata && this.isExpired(currentTime, localMetadata.updated)) {
{'fraction_non_alphanumeric': 0.11282011843420749, 'fraction_numerical': 0.00969007152195647, 'mean_word_length': 3.256628477905074, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2995512', 'n_tokens_mistral': 3728, 'n_tokens_neox': 3613, 'n_words': 1023}