<commit_before>/*    Copyright 2016 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('./logger.js')(__filename);

const util = require('util');

const FlowAggrTool = require('./FlowAggrTool');
const flowAggrTool = new FlowAggrTool();

const HostTool = require('./HostTool');
const hostTool = new HostTool();

const AppFlowTool = require('../flow/AppFlowTool.js')
const appFlowTool = new AppFlowTool()

const flowTool = require('./FlowTool.js')();

const CategoryFlowTool = require('../flow/CategoryFlowTool.js')
const categoryFlowTool = new CategoryFlowTool()

const rclient = require('../util/redis_manager.js').getRedisClient()

let instance = null;

function toInt(n){ return Math.floor(Number(n)); };


class NetBotTool {
  constructor() {
    if(!instance) {
      instance = this;
    }
    return instance;
  }

  async loadSystemStats(json) {
    const systemFlows = {};

    const keys = ['upload', 'download'];

    for(const key of keys) {
      const lastSumKey = `lastsumflow:${key}`;
      const realSumKey = await rclient.getAsync(lastSumKey);
      const elements = realSumKey.split(":")
      if(elements.length !== 4) {
        continue;
      }

      const begin = elements[2];
      const end = elements[3];

      const traffic = await flowAggrTool.getTopSumFlowByKeyAndDestination(realSumKey, 50);

      const enriched = (await flowTool.enrichWithIntel(traffic)).sort((a, b) => {
        return b.count - a.count;
      });

      systemFlows[key] = {
        begin,
        end,
        flows: enriched
      }
    }

    const actitivityKeys = ['app', 'category'];

    for(const key of actitivityKeys) {

      const lastSumKey = `lastsumflow:${key}`;
      const realSumKey = await rclient.getAsync(lastSumKey);
      const elements = realSumKey.split(":")
      if(elements.length !== 4) {
        continue;
      }

      const begin = elements[2];
      const end = elements[3];
  
      const traffic = await flowAggrTool.getXYActivitySumFlowByKey(realSumKey, key, 50);
        
      traffic.sort((a, b) => {
        return b.count - a.count;
      });

      systemFlows[key] = {
        begin,
        end,
        activities: traffic
      }  
    }

    json.systemFlows = systemFlows;
  }

  prepareTopDownloadFlows(json, options) {
    return this._prepareTopFlows(json, "download", options);
  }

  prepareTopUploadFlows(json, options) {
    return this._prepareTopFlows(json, "upload", options);
  }<commit_msg>bug if lastsum key doesn't exist<commit_after>/*    Copyright 2016 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('./logger.js')(__filename);

const util = require('util');

const FlowAggrTool = require('./FlowAggrTool');
const flowAggrTool = new FlowAggrTool();

const HostTool = require('./HostTool');
const hostTool = new HostTool();

const AppFlowTool = require('../flow/AppFlowTool.js')
const appFlowTool = new AppFlowTool()

const flowTool = require('./FlowTool.js')();

const CategoryFlowTool = require('../flow/CategoryFlowTool.js')
const categoryFlowTool = new CategoryFlowTool()

const rclient = require('../util/redis_manager.js').getRedisClient()

let instance = null;

function toInt(n){ return Math.floor(Number(n)); };


class NetBotTool {
  constructor() {
    if(!instance) {
      instance = this;
    }
    return instance;
  }

  async loadSystemStats(json) {
    const systemFlows = {};

    const keys = ['upload', 'download'];

    for(const key of keys) {
      const lastSumKey = `lastsumflow:${key}`;
      const realSumKey = await rclient.getAsync(lastSumKey);
      if(!realSumKey) {
        continue;
      }

      const elements = realSumKey.split(":")
      if(elements.length !== 4) {
        continue;
      }

      const begin = elements[2];
      const end = elements[3];

      const traffic = await flowAggrTool.getTopSumFlowByKeyAndDestination(realSumKey, 50);

      const enriched = (await flowTool.enrichWithIntel(traffic)).sort((a, b) => {
        return b.count - a.count;
      });

      systemFlows[key] = {
        begin,
        end,
        flows: enriched
      }
    }

    const actitivityKeys = ['app', 'category'];

    for(const key of actitivityKeys) {

      const lastSumKey = `lastsumflow:${key}`;
      const realSumKey = await rclient.getAsync(lastSumKey);
      if(!realSumKey) {
        continue;
      }
      
      const elements = realSumKey.split(":")
      if(elements.length !== 4) {
        continue;
      }

      const begin = elements[2];
      const end = elements[3];
  
      const traffic = await flowAggrTool.getXYActivitySumFlowByKey(realSumKey, key, 50);
        
      traffic.sort((a, b) => {
        return b.count - a.count;
      });

      systemFlows[key] = {
        begin,
        end,
        activities: traffic
      }  
    }

    json.systemFlows = systemFlows;
  }

  prepareTopDownloadFlows(json, options) {
    return this._prepareTopFlows(json, "download", options);
  }

  prepareTopUploadFlows(json, options) {
    return this._prepareTopFlows(json, "upload", options);
  }
{'fraction_non_alphanumeric': 0.11681724520526232, 'fraction_numerical': 0.004755111745126011, 'mean_word_length': 2.866421568627451, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '612275', 'n_tokens_mistral': 1900, 'n_tokens_neox': 1852, 'n_words': 527}