<commit_before>#!/usr/bin/env node
/*    Copyright 2016 Firewalla LLC / Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

'use strict'

process.title = "FireApi";
let log = require('../net2/logger.js')(__filename, "info");

const util = require('util');
let fs = require('fs');

const ControllerBot = require('../lib/ControllerBot.js');

const sem = require('../sensor/SensorEventManager.js').getInstance();

let HostManager = require('../net2/HostManager.js');
let SysManager = require('../net2/SysManager.js');
let FlowManager = require('../net2/FlowManager.js');
let flowManager = new FlowManager('info');
let AlarmManager = require('../net2/AlarmManager.js');
let alarmManager = new AlarmManager('info');
let sysManager = new SysManager();
let VpnManager = require("../vpn/VpnManager.js");
let vpnManager = new VpnManager('info');
let IntelManager = require('../net2/IntelManager.js');
let intelManager = new IntelManager('debug');

let DeviceMgmtTool = require('../util/DeviceMgmtTool');

const Promise = require('bluebird');

const SysTool = require('../net2/SysTool.js')
const sysTool = new SysTool()

const flowUtil = require('../net2/FlowUtil');

const iptool = require('ip')

let redis = require('redis');
let rclient = redis.createClient();
var sclient = redis.createClient();
sclient.setMaxListeners(0);

Promise.promisifyAll(redis.RedisClient.prototype);

const exec = require('child-process-promise').exec

let AM2 = require('../alarm/AlarmManager2.js');
let am2 = new AM2();

let EM = require('../alarm/ExceptionManager.js');
let em = new EM();

let PM2 = require('../alarm/PolicyManager2.js');
let pm2 = new PM2();

let SSH = require('../extension/ssh/ssh.js');
let ssh = new SSH('info');

let country = require('../extension/country/country.js');

let builder = require('botbuilder');
let uuid = require('uuid');

let async2 = require('async');

let async = require('asyncawait/async');
let await = require('asyncawait/await');

let NM = require('../ui/NotifyManager.js');
let nm = new NM();

let FRP = require('../extension/frp/frp.js')
let frp = new FRP();

let f = require('../net2/Firewalla.js');

let flowTool = require('../net2/FlowTool')();

let i18n = require('../util/i18n');

let NetBotTool = require('../net2/NetBotTool');
let netBotTool = new NetBotTool();

let HostTool = require('../net2/HostTool');
let hostTool = new HostTool();

let appTool = require('../net2/AppTool')();

let spooferManager = require('../net2/SpooferManager.js')

const extMgr = require('../sensor/ExtensionManager')

class netBot extends ControllerBot {

  _block2(ip, dst, cron, timezone, duration, callback) {
    let value = {
      id: "0",
      cron: cron,
      dst: dst,
      timezone: timezone,
      duration: duration
    }
    this._block(ip, 'block', value, callback);
  }

  _ignore(ip, reason, callback) {
    this.hostManager.ignoreIP(ip, reason, callback);
  }

  _unignore(ip, callback) {
    this.hostManager.unignoreIP(ip, callback);
  }

  _block(ip, blocktype, value, callback) {
    log.info("_block", ip, blocktype, value);
    if (ip === "0.0.0.0") {
      this.hostManager.loadPolicy((err, data) => {
        this.hostManager.setPolicy(blocktype, value, (err, data) => {
          if (err == null) {
            if (callback != null)
              callback(null, "Success");
          } else {
            if (callback != null)
              callback(err, "Unable to block ip " + ip);
          }
        });
      });
    } else {
      this.hostManager.getHost(ip, (err, host) => {
        if (host != null) {
          host.loadPolicy((err, data) => {
            if (err == null) {
              host.setPolicy(blocktype, value, (err, data) => {
                if (err == null) {
                  if (callback != null)
                  //   this.tx(this.primarygid, "Success:"+ip,"hosts summary");
                    callback(null, "Success:" + ip);
                } else {
                  if (callback != null)
                  // this.tx(this.primarygid, "Unable to block ip "+ip,"hosts summary");
                    callback(err, "Unable to block ip " + ip)

                }
              });
            } else {
              if (callback != null)
              //this.tx(this.primarygid, "Unable to block ip "+ip,"hosts summary");
                callback("error", "Unable to block ip " + ip);
            }
          });
        } else {
          if (callback != null)
          //this.tx(this.primarygid, "host not found","hosts summary");
            callback("error", "Host not found");
        }
      });
    }
  }

  _family(ip, value, callback) {
    if (ip === "0.0.0.0") {
      this.hostManager.loadPolicy((err, data) => {
        this.hostManager.setPolicy("family", value, (err, data) => {
          if (err == null) {
            if (callback != null)
              callback(null, "Success");
          } else {
            if (callback != null)
              callback(err, "Unable to block ip " + ip);
          }
        });
      });
    } else {
      this.hostManager.getHost(ip, (err, host) => {
        if (host != null) {
          host.loadPolicy((err, data) => {
            if (err == null) {
              host.setPolicy("family", value, (err, data) => {
                if (err == null) {
                  if (callback != null)
                    callback(null, "Success:" + ip);
                } else {
                  if (callback != null)
                    callback(err, "Unable to block ip " + ip);
                }
              });
            } else {
              if (callback != null)
                callback("error", "Unable to block ip " + ip);
            }
          });
        } else {
          if (callback != null)
            callback("error", "host not found");
        }
      });
    }
  }

  _adblock(ip, value, callback) {
    if (ip === "0.0.0.0") {
      this.hostManager.loadPolicy((err, data) => {
        this.hostManager.setPolicy("adblock", value, (err, data) => {
          if (err == null) {
            if (callback != null)
              callback(null, "Success");
          } else {
            if (callback != null)
              callback(err, "Unable to block ip " + ip);
          }
        });
      });
    } else {
      this.hostManager.getHost(ip, (err, host) => {
        if (host != null) {
          host.loadPolicy((err, data) => {
            if (err == null) {
              host.setPolicy("adblock", value, (err, data) => {
                if (err == null) {
                  if (callback != null)
                    callback(null, "Success:" + ip);
                } else {
                  if (callback != null)
                    callback(err, "Unable to block ip " + ip);
                }
              });
            } else {
              if (callback != null)
                callback("error", "Unable to block ip " + ip);
            }
          });
        } else {
          if (callback != null)
            callback("error", "host not found");
        }
      });
    }
  }

  _vpn(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("vpn", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to block ip " + ip);
        }
      });
    });
  }

  _shadowsocks(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("shadowsocks", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to apply config on shadowsocks: " + value);
        }
      });
    });
  }

  _scisurf(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("scisurf", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to apply config on scisurf: " + value);
        }
      });
    });
  }

  _vulScan(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("vulScan", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to apply config on vulScan: " + value);
        }
      });
    });
  }

  _dnsmasq(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("dnsmasq", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to apply config on dnsmasq: " + value);
        }
      });
    });
  }

  _externalAccess(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("externalAccess", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to apply config on externalAccess: " + value);
        }
      });
    });
  }

  _ssh(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("ssh", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to ssh " + ip);
        }
      });
    });
  }

  /*
   *
   *   {
   *      state: BOOL;  overall notification
   *      ALARM_XXX: standard alarm definition
   *      ALARM_BEHAVIOR: may be mapped to other alarms
   *   }
   */
  _notify(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("notify", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to setNotify " + ip);
        }
        log.info("Notification Set",value," CurrentPolicy:", JSON.stringify(this.hostManager.policy.notify),{});
        nm.loadConfig();
      });
    });
  }

  constructor(config, fullConfig, eptcloud, groups, gid, debug, apiMode) {
    super(config, fullConfig, eptcloud, groups, gid, debug, apiMode);
    this.bot = new builder.TextBot();
    //      this.dialog = new builder.LuisDialog(config.dialog.api);
    this.dialog = new builder.CommandDialog();
    this.bot.add('/', this.dialog);
    let self = this;
    this.compress = true;
    this.scanning = false;

    this.sensorConfig = config.controller.sensor;
    //flow.summaryhours
    // sysManager.setConfig(this.sensorConfig);
    sysManager.update((err, data) => {
    });

    setInterval(() => {
      sysManager.update((err, data) => {
      });
    }, 1000 * 60 * 60 * 10);

    setInterval(() => {
      try {
        if (global.gc) {
          global.gc();
        }
      } catch (e) {
      }
    }, 1000 * 60);

    setTimeout(() => {
      setInterval(() => {
//          this.refreshCache(); // keep cache refreshed every 50 seconds so that app will load data fast
      }, 50 * 1000);
    }, 30 * 1000)

    this.hostManager = new HostManager("cli", 'client', 'debug');
    this.hostManager.loadPolicy((err, data) => {});  //load policy
 
    // no subscription for api mode
    if (apiMode) {
      log.info("Skipping event subscription during API mode.");
      return;
    }

    let c = require('../net2/MessageBus.js');
    this.subscriber = new c('debug');

    this.subscriber.subscribe("DiscoveryEvent", "DiscoveryStart", null, (channel, type, ip, msg) => {
      //this.tx(this.primarygid, "Discovery started","message");
    });
    this.subscriber.subscribe("DiscoveryEvent", "Host:Found", null, (channel, type, ip, o) => {
      log.info("Found new host ", channel, type, ip);
      if (o) {
        let name = o.ipv4Addr;
        if (o.name != null) {
          name = o.name + " (" + o.ipv4Addr + ")";
        } else if (o.macVendor != null) {
          name = "(?)" + o.macVendor + " (" + o.ipv4Addr + ")";
        }
        this.tx2(this.primarygid, "New host found in network: " + name, "Found new host " + name, {uid: o.ipv4Addr});
      }
    });
    this.subscriber.subscribe("MonitorEvent", "Monitor:Flow:Out", null, (channel, type, ip, msg) => {
      let m = null;
      let n = null;
      log.info("Monitor:Flow:Out", channel, ip, msg, "=====");
      if (ip && msg) {
        if (msg['txRatioRanked'] && msg['txRatioRanked'].length > 0) {
          let flow = msg['txRatioRanked'][0];
          if (flow.rank > 0) {
            return;
          }
          m = "Warning: \n\n" + flowManager.toStringShortShort2(msg['txRatioRanked'][0], msg.direction, 'txdata') + "\n";
          n = flowManager.toStringShortShort2(msg['txRatioRanked'][0], msg.direction);
        }
      }
      if (m) {
        log.info("MonitorEvent:Flow:Out", m, msg);
        if (nm.canNotify() == true) {
          this.tx2(this.primarygid, m, n, {id: msg.id});
        }
      }
    });
    this.subscriber.subscribe("MonitorEvent", "Monitor:Flow:In", null, (channel, type, ip, msg) => {
      /*
       let m = null;
       let n = null;
       log.info("Monitor:Flow:In", channel, ip, msg, "=====");
       if (ip && msg) {
       if (msg['txRatioRanked'] && msg['txRatioRanked'].length > 0) {
       let flow = msg['txRatioRanked'][0];
       if (flow.rank > 0) {
       return;
       }
       m = "Warning: \n\n" + flowManager.toStringShortShort2(msg['txRatioRanked'][0], msg.direction, 'txdata') + "\n";
       n = flowManager.toStringShortShort2(msg['txRatioRanked'][0], msg.direction);
       }
       }
       if (m)
       this.tx2(this.primarygid, m, n, {id:msg.id});
       */
    });


    this.subscriber.subscribe("ALARM", "ALARM:CREATED", null, (channel, type, ip, msg) => {
      if (msg) {
        let notifMsg = msg.notif;
        let aid = msg.aid;
        if (notifMsg) {
          log.info("Sending notification: " + JSON.stringify(msg));
          if (this.hostManager.policy && this.hostManager.policy["notify"]) {
               if (this.hostManager.policy['notify']['state']==false) {
                   log.info("ALARM_NOTIFY_GLOBALL_BLOCKED", msg);
                   return;
               }
               if (msg.alarmType) {
                   let alarmType = msg.alarmType;
                   if (msg.alarmType  === "ALARM_LARGE_UPDATE") {
                       alarmType = "ALARM_BEHAVIOR";
                   }
                   if (this.hostManager.policy["notify"][alarmType] === false) {
                       log.info("ALARM_NOTIFY_BLOCKED", msg);
                       return;
                   }
               }
          }

          log.info("ALARM_NOTIFY_PASSED");

          notifMsg = {
            title: i18n.__("SECURITY_ALERT"),
            body: notifMsg
          }

          let data = {
            gid: this.primarygid,
          };

          if (msg.aid) {
            data.aid = msg.aid;
          }

          if (msg.alarmID) {
            data.alarmID = msg.alarmID;
          }

          switch(msg.alarmNotifType) {
            case "security":
              notifMsg.title = i18n.__("SECURITY_ALERT");
              break;
            case "activity":
              notifMsg.title = i18n.__("ACTIVITY_ALERT");
              break;
            default:
              break;
          }

          if (msg.autoblock) {
            data.category = "com.firewalla.category.autoblockalarm";
          } else {
            data.category = "com.firewalla.category.alarm";
          }

          // check if device name should be included, sometimes it is helpful if multiple devices are bound to one app
          async(() => {
            let flag = await (rclient.hgetAsync("sys:config", "includeNameInNotification"))
            if(flag) {
              notifMsg.body = `[${this.getDeviceName()}] ${notifMsg.body}`
            }
            this.tx2(this.primarygid, "test", notifMsg, data);            
          })()


        }
      }
    });

    setTimeout(() => {
      this.scanStart();
      async(() => {
        let branchChanged = await (sysManager.isBranchJustChanged())
        if(branchChanged) {
          let branch = null
          
          switch(branchChanged) {
          case "1":
            branch = "back to stable version"
            break;
          case "2":
            branch = "to pre_release version"
            break;
          case "3":
            branch = "to development version"
            break;
          default:
            // do nothing, should not happen here
            break;
          }

          if(branch) {
            let msg = `Device '${this.getDeviceName()}' has switched ${branch} ${sysManager.version()} successfully`
            log.info(msg)
            this.tx(this.primarygid, "200", msg)
            sysManager.clearBranchChangeFlag()            
          }

        } else {
          if (sysManager.systemRebootedDueToIssue(true) == false) {
            if (nm.canNotify() == true) {
              this.tx(this.primarygid, "200", "🔥 Firewalla Device '" + this.getDeviceName() + "' Awakens!");
            }
          }
        }
        
      })()
      this.setupDialog();
    }, 2000);

    this.hostManager.on("Scan:Done", (channel, type, ip, obj) => {
      if (type == "Scan:Done") {
        this.scanning = false;
        for (let h in this.hosts) {
          //this.hosts[h].clean();
        }
        this.scanStart();
      }
    });
    this.hostManager.on("Scan:Start", (channel, type, ip, obj) => {
      if (type == "Scan:Start") {
        this.scanning = true;
      }
    });

    sclient.on("message", (channel, msg)=> {
       log.info("Msg",channel,msg);
       switch(channel) {
         case "System:Upgrade:Hard":
             if (msg) {
                let notifyMsg = {
                   title: "Upgrade Needed",
                   body: "Firewalla new version "+msg+" is avaliable, please open firewalla app then tap on settings->upgrade.",
                   } 
                let data = {
                   gid: this.primarygid,
                };
                this.tx2(this.primarygid, "", notifyMsg, data);
             }
             break;
         case "System:Upgrade:Soft":
             if (msg) {
                let notifyMsg = {
                  title: `Firewalla is upgraded to ${msg}`,
                  body: ""
                }
                let data = {
                  gid: this.primarygid,
                };
                this.tx2(this.primarygid, "", notifyMsg, data);
             }
             break;
       }
    });
    sclient.subscribe("System:Upgrade:Hard");
    sclient.subscribe("System:Upgrade:Soft");


  }

  boneMsgHandler(msg) {
      if (msg.type == "MSG" && msg.title) {
          let notifyMsg = {
             title: msg.title,
             body: msg.body 
          } 
          let data = {
             gid: this.primarygid,
             data: msg.data
          };
          this.tx2(this.primarygid, "", notifyMsg, data);
      }
  }

  scanStart(callback) {
    this.hostManager.getHosts((err, result) => {
      let listip = [];
      this.hosts = result;
      for (let i in result) {
//        log.info(result[i].toShortString());
        result[i].on("Notice:Detected", (channel, type, ip, obj) => {
          log.info("Found new notice", type, ip);
          if ((obj.note == "Scan::Port_Scan" || obj.note == "Scan::Address_Scan") && this.scanning == false) {
            let msg = result[i].name() + ": " + obj.msg;
            if (nm.canNotify() == true) {
              this.tx(this.primarygid, msg, obj.msg);
            }
          } else if ((obj.note == "Scan::Port_Scan" || obj.note == "Scan::Address_Scan") && this.scanning == true) {
            log.info("Netbot:Notice:Skip due to scanning", obj);
          } else {
            let msg = result[i].name() + ": " + obj.msg;
            if (nm.canNotify() == true) {
              this.tx(this.primarygid, msg, obj.msg);
            }
          }
        });
      }
      if (callback)
        callback(null, null);

    });

  }

  setHandler(gid, msg, callback) {
    // mtype: set
    // target = "ip address" 0.0.0.0 is self
    // data.item = policy
    // data.value = {'block':1},
    //
    //       log.info("Set: ",gid,msg);

    // invalidate cache
    this.invalidateCache();


    switch (msg.data.item) {
      case "policy":
        async2.eachLimit(Object.keys(msg.data.value), 1, (o, cb) => {
          switch (o) {
            case "monitor":
              this._block(msg.target, "monitor", msg.data.value.monitor, (err, obj) => {
                cb(err);
              });
              break;
            case "blockin":
              this._block(msg.target, "blockin", msg.data.value.blockin, (err, obj) => {
                cb(err);
              });
              break;
            case "acl":
              this._block(msg.target, "acl", msg.data.value.acl, (err, obj) => {
                cb(err);
              });
              break;
            case "family":
              this._family(msg.target, msg.data.value.family, (err, obj) => {
                cb(err);
              });
              break;
            case "adblock":
              this._adblock(msg.target, msg.data.value.adblock, (err, obj) => {
                cb(err);
              });
              break;
            case "vpn":
              this._vpn(msg.target, msg.data.value.vpn, (err, obj) => {
                cb(err);
              });
              break;
            case "shadowsocks":
              this._shadowsocks(msg.target, msg.data.value.shadowsocks, (err, obj) => {
                cb(err);
              });
              break;
            case "scisurf":
              this._scisurf(msg.target, msg.data.value.scisurf, (err, obj) => {
                cb(err);
              });
              break;
            case "vulScan":
              this._vulScan(msg.target, msg.data.value.vulScan, (err, obj) => {
                cb(err);
              });
              break;
            case "dnsmasq":
              this._dnsmasq(msg.target, msg.data.value.dnsmasq, (err, obj) => {
                cb(err);
              });
              break;
            case "externalAccess":
              this._externalAccess(msg.target, msg.data.value.externalAccess, (err, obj) => {
                cb(err);
              });
              break;
            case "ssh":
              this._ssh(msg.target, msg.data.value.ssh, (err, obj) => {
                cb(err);
              });
              break;
            case "ignore":
              this._ignore(msg.target, msg.data.value.ignore.reason, (err, obj) => {
                cb(err);
              });
              break;
            case "unignore":
              this._unignore(msg.target, (err, obj) => {
                cb(err);
              });
              break;
            case "notify":
              this._notify(msg.target, msg.data.value.notify, (err, obj) => {
                cb(err);
              });
              break;
          default:
            let target = msg.target
            let policyData = msg.data.value[o]

            //            if(extMgr.hasExtension(o)) {
              this.hostManager.loadPolicy((err, data) => {
                this.hostManager.setPolicy(o,
                                           policyData,
                                           (err, data) => {
                  cb(err)
                })
              })
            // } else {
            //   cb(null)
            // }
            break
          }
        }, (err) => {
          let reply = {
            type: 'jsonmsg',
            mtype: 'policy',
            id: uuid.v4(),
            expires: Math.floor(Date.now() / 1000) + 60 * 5,
            replyid: msg.id,
          };
          reply.code = 200;
          reply.data = msg.data.value;
          log.info("Repling ", reply.code, reply.data);
          this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);

        });
        break;
      case "host":
        //data.item = "host" test
        //data.value = "{ name: " "}"
        let data = msg.data;
        log.info("Setting Host", msg);
        let reply = {
          type: 'jsonmsg',
          mtype: 'init',
          id: uuid.v4(),
          expires: Math.floor(Date.now() / 1000) + 60 * 5,
          replyid: msg.id,
        };
        reply.code = 200;

        this.hostManager.getHost(msg.target, (err, host) => {
          if (host == null) {
            log.info("Host not found");
            reply.code = 404;
            this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);
            return;
          }

          if (data.value.name == host.o.name) {
            log.info("Host not changed", data.value.name, host.o.name);
            reply.code = 200;
            this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);
            return;
          }

          host.o.name = data.value.name;
          log.info("Changing names", host.o.name);
          host.save(null, (err) => {
            if (err) {
              reply.code = 500;
              this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);
            } else {
              reply.code = 200;
              reply.data = msg.data.value;
              this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);
            }
          });
        });
        break;
      case "intel":
        // intel actions
        //   - ignore / unignore
        //   - report
        //   - block / unblockj
        intelManager.action(msg.target, msg.data.value.action, (err) => {
          let reply = {
            type: 'jsonmsg',
            mtype: 'init',
            id: uuid.v4(),
            expires: Math.floor(Date.now() / 1000) + 60 * 5,
            replyid: msg.id,
          };
          reply.code = 200;
          this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);
        });
        break;
      case "scisurfconfig":
        let v = msg.data.value;

        // TODO validate input ??
        if (v.from && v.from === "firewalla") {
          let scisurf = require('../extension/ss_client/ss_client.js');
          scisurf.saveConfig(v, (err) => {
            this.simpleTxData(msg, {}, err, callback);
          });
        } else {
          this.simpleTxData(msg, {}, new Error("Invalid config"), callback);
        }

        break;
      case "language":
        let v2 = msg.data.value;

        // TODO validate input?
        if (v2.language) {
          sysManager.setLanguage(v2.language, (err) => {
            this.simpleTxData(msg, {}, err, callback);
          });
        }
        break;
      case "timezone":
        let v3 = msg.data.value;

        if (v3.timezone) {
          sysManager.setTimezone(v3.timezone, (err) => {
            this.simpleTxData(msg, {}, err, callback);
          });
        }
      break;
    case "includeNameInNotification":
        let v33 = msg.data.value;

      if (v3.includeNameInNotification) {
        async(() => {
          await (rclient.hsetAsync("sys:config", "includeNameInNotification", "1"))
          this.simpleTxData(msg, {}, null, callback)
        })().catch((err) => {
          this.simpleTxData(msg, {}, err, callback)
        })
      }
      break;
    case "mode":
      let v4 = msg.data.value;
      let err = null;
      
      if (v4.mode) {
        async(() => {
          let mode = require('../net2/Mode.js')
          let curMode = await (mode.getSetupMode())        
          if(v4.mode === curMode) {
            this.simpleTxData(msg, {}, err, callback);
            return
          }
          
          let modeManager = require('../net2/ModeManager.js');
          switch (v4.mode) {
          case "spoof":
          case "autoSpoof":
            modeManager.setAutoSpoofAndPublish()
            break;
          case "manualSpoof":
            modeManager.setManualSpoofAndPublish()
            break;
          case "dhcp":
            modeManager.setDHCPAndPublish()
            break;
          case "none":
            modeManager.setNoneAndPublish()
            break;
          default:
            log.error("unsupported mode: " + v4.mode);
            err = new Error("unsupport mode: " + v4.mode);
            break;
          }
          this.simpleTxData(msg, {}, err, callback);
        })().catch((err) => {
          this.simpleTxData(msg, {}, err, callback);
        })
      }
      break;
    default:
        this.simpleTxData(msg, null, new Error("Unsupported action"), callback);
        break;
    }
  }


  getAllIPForHost(ip, callback) {
    let listip = [];
    this.hostManager.getHost(ip, (err, host) => {
      if (host != null) {
        listip.push(host.o.ipv4Addr);
        if (host.ipv6Addr && host.ipv6Addr.length > 0) {
          for (let j in host['ipv6Addr']) {
            listip.push(host['ipv6Addr'][j]);
          }
        }
      }
      callback(err, listip);
    });
  }




  getHandler(gid, msg, appInfo, callback) {

    // backward compatible
    if(typeof appInfo === 'function') {
      callback = appInfo;
      appInfo = undefined;
    }

    // mtype: get
    // target = ip address
    // data.item = [app, alarms, host]

    switch (msg.data.item) {
      case "host":
        if (msg.target) {
          let useNewDeviceHandler = appTool.isAppReadyForNewDeviceHandler(appInfo);
          if(useNewDeviceHandler) {
            let ip = msg.target;
            log.info("Loading device info in a new way:", ip);
            this.newDeviceHandler(msg, ip)
            .then((json) => {
              this.simpleTxData(msg, json, null, callback);
            })
            .catch((err) => {
              this.simpleTxData(msg, null, err, callback);
            })
          } else {
            log.info("Using the legacy way to get device info:", msg.target, {});
            this.getAllIPForHost(msg.target, (err, ips) => {
              this.deviceHandler(msg, gid, msg.target, ips, callback);
            });
          }
        }
        break;
      case "vpn":
      case "vpnreset":
        let regenerate = true;
        if (msg.data.item === "vpnreset") {
          regenerate = true;
        }

        this.hostManager.loadPolicy(() => {
          vpnManager.getOvpnFile("fishboneVPN1", null, regenerate, (err, ovpnfile, password) => {
            let datamodel = {
              type: 'jsonmsg',
              mtype: 'reply',
              id: uuid.v4(),
              expires: Math.floor(Date.now() / 1000) + 60 * 5,
              replyid: msg.id,
              code: 404,
            };
            if (err == null) {
              datamodel.code = 200;
              datamodel.data = {
                ovpnfile: ovpnfile,
                password: password,
                portmapped: this.hostManager.policy['vpnPortmapped']
              }
            }
            this.txData(this.primarygid, "device", datamodel, "jsondata", "", null, callback);
          });
        });
        break;
      case "shadowsocks":
      case "shadowsocksResetConfig":
        let shadowsocks = require('../extension/shadowsocks/shadowsocks.js');
        let ss = new shadowsocks('info');

        if (msg.data.item === "shadowsocksResetConfig") {
          ss.refreshConfig();
        }

        let config = ss.readConfig();
        let datamodel = {
          type: 'jsonmsg',
          mtype: 'reply',
          id: uuid.v4(),
          expires: Math.floor(Date.now() / 1000) + 60 * 5,
          replyid: msg.id,
          code: 200,
          data: {
            config: config
          }
        };
        this.txData(this.primarygid, "device", datamodel, "jsondata", "", null, callback);
        break;
      case "sshPrivateKey":

        ssh.getPrivateKey((err, data) => {
          if (err) {
            log.error("Got error when loading ssh private key: " + err);
            data = "";
          }

          let datamodel = {
            type: 'jsonmsg',
            mtype: 'reply',
            id: uuid.v4(),
            expires: Math.floor(Date.now() / 1000) + 60 * 5,
            replyid: msg.id,
            code: 200,
            data: {
              key: data
            }
          };
          this.txData(this.primarygid, "device", datamodel, "jsondata", "", null, callback);
        });
        break;
      case "sshRecentPassword":
        ssh.getPassword((err, password) => {

          let data = "";

          if (err) {
            log.error("Got error when reading password: " + err);
            this.simpleTxData(msg, {}, err, callback);
          } else {
            this.simpleTxData(msg, {password: password}, err, callback);
          }
        });
        break;
      case "sysInfo":
        let si = require('../extension/sysinfo/SysInfo.js');
        this.simpleTxData(msg, si.getSysInfo(), null, callback);
        break;
      case "logFiles":
        let si2 = require('../extension/sysinfo/SysInfo.js');
        si2.getRecentLogs((err, results) => {
          this.simpleTxData(msg, results, null, callback);
        });
        break;
      case "scisurfconfig":
        let ssc = require('../extension/ss_client/ss_client.js');
        ssc.loadConfig((err, result) => {
          this.simpleTxData(msg, result || {}, err, callback);
        });
        break;
      case "language":
        this.simpleTxData(msg, {language: sysManager.language}, null, callback);
        break;
      case "timezone":
        this.simpleTxData(msg, {timezone: sysManager.timezone}, null, callback);
        break;
      case "alarms":
        am2.loadActiveAlarms((err, alarms) => {
          this.simpleTxData(msg, {alarms: alarms, count: alarms.length}, err, callback);
        });
        break;
      case "alarm":
        let alarmID = msg.data.value.alarmID;
        am2.getAlarm(alarmID)
          .then((alarm) => this.simpleTxData(msg, alarm, null, callback))
          .catch((err) => this.simpleTxData(msg, null, err, callback));
        break;
      case "exceptions":
        em.loadExceptions((err, exceptions) => {
          this.simpleTxData(msg, {exceptions: exceptions, count: exceptions.length}, err, callback);
        });
        break;
    case "frpConfig":
      let _config = frp.getConfig()
      if(_config.started) {
        let getPasswordAsync = Promise.promisify(ssh.getPassword)
        getPasswordAsync().then((password) => {
          _config.password = password
          this.simpleTxData(msg, _config, null, callback);
        }).catch((err) => {
          this.simpleTxData(msg, null, err, callback);
        })
      } else {
        this.simpleTxData(msg, _config, null, callback);
      }
      break;
    default:
        this.simpleTxData(msg, null, new Error("unsupported action"), callback);
    }
  }

  validateFlowIntel(json) {
    return async(() => {
      // await (bone.flowgraphAsync(...))
      let flows = json.flows

      let hashCache = {}


      let appFlows = flows.appDetails
      let categoryFlows = flows.categoryDetails

      if(Object.keys(appFlows).length > 0 ||
         Object.keys(categoryFlows).length > 0) {
        flowUtil.hashIntelFlows(appFlows, hashCache)
        flowUtil.hashIntelFlows(categoryFlows, hashCache)
        
        let data = await (bone.flowgraphAsync('summarizeApp', appFlows),
                          bone.flowgraphAsync('summarizeActivity', categoryFlows))

        let unhashedData = flowUtil.unhashIntelFlows(data[0], hashCache)
        let unhashedData2 = flowUtil.unhashIntelFlows(data[1], hashCache)
        
        flows.appDetails = unhashedData
        flows.categoryDetails = unhashedData2
      }
    })()
  }
  
  systemFlowHandler(msg) {
    log.info("Getting flow info of the entire network");

    let begin = msg.data && msg.data.start;
    //let end = msg.data && msg.data.end;
    let end = begin && (begin + 3600);

    if(!begin || !end) {
      return Promise.reject(new Error("Require begin and error when calling systemFlowHandler"));
    }

    log.info("FROM: ", new Date(begin * 1000).toLocaleTimeString());
    log.info("TO: ", new Date(end * 1000).toLocaleTimeString());

    return async(() => {
      let jsonobj = {};
      let options = {
        begin: begin,
        end: end
      }
      await (flowTool.prepareRecentFlows(jsonobj, options))
      await (netBotTool.prepareTopUploadFlows(jsonobj, options))
      await (netBotTool.prepareTopDownloadFlows(jsonobj, options))

      await (netBotTool.prepareDetailedAppFlowsFromCache(jsonobj, options))
      if(!jsonobj.flows['appDetails']) { // fallback to old way
        await (netBotTool.prepareDetailedAppFlows(jsonobj, options))
        await (this.validateFlowIntel(jsonobj))
      }

      await (netBotTool.prepareDetailedCategoryFlowsFromCache(jsonobj, options))
      if(!jsonobj.flows['categoryDetails']) { // fallback to old model
        await (netBotTool.prepareDetailedCategoryFlows(jsonobj, options))
        await (this.validateFlowIntel(jsonobj))
      }

      return jsonobj;
    })();
  }

  newDeviceHandler(msg, ip) {
    log.info("Getting info on device", ip, {});

    return async(() => {
      if(ip === '0.0.0.0') {
        return this.systemFlowHandler(msg);
      }

      let begin = msg.data && msg.data.start;
      let end = (msg.data && msg.data.end) || begin + 3600 * 24;

      let options = {}
      if(begin && end) {
        options.begin = begin
        options.end = end
      }

      if(msg.data.hourblock != "1" &&
         msg.data.hourblock != "0" ) { // 0 => now, 1 => hour stats, other => overall stats (last 24 hours)
        options.queryall = true
      }
      
      let host = await (this.hostManager.getHostAsync(ip));
      if(!host || !host.o.mac) {
        let error = new Error("Invalide Host");
        error.code = 404;
        return Promise.reject(error);
      }

      let mac = host.o.mac;

      // load 24 hours download/upload trend
      await (flowManager.getStats2(host));

      let jsonobj = {};
      if (host) {
        jsonobj = host.toJson();

        await (flowTool.prepareRecentFlowsForHost(jsonobj, mac, options));
        await (netBotTool.prepareTopUploadFlowsForHost(jsonobj, mac, options));
        await (netBotTool.prepareTopDownloadFlowsForHost(jsonobj, mac, options));
        
        await (netBotTool.prepareAppActivityFlowsForHostFromCache(jsonobj, mac, options))
        if(!jsonobj.flows["appDetails"]) {
          log.warn("Fell back to legacy mode on app details:", mac, options, {})
          await (netBotTool.prepareAppActivityFlowsForHost(jsonobj, mac, options))
          await (this.validateFlowIntel(jsonobj))
        }
        
        await (netBotTool.prepareCategoryActivityFlowsForHostFromCache(jsonobj, mac, options))
        if(!jsonobj.flows["categoryDetails"]) {
          log.warn("Fell back to legacy mode on category details:", mac, options, {})
          await (netBotTool.prepareCategoryActivityFlowsForHost(jsonobj, mac, options))
          await (this.validateFlowIntel(jsonobj))
        }
        
        await (netBotTool.prepareDetailedAppFlowsForHost(jsonobj, mac, options))        
        await (netBotTool.prepareDetailedCategoryFlowsForHost(jsonobj, mac, options))
      }

      return jsonobj;
    })();
  }

  deviceHandler(msg, gid, target, listip, callback) {
    log.info("Getting Devices", gid, target, listip);
    let hosts = [];
    this.hostManager.getHost(target, (err, host) => {
      if (host == null && target != "0.0.0.0") {
        let datamodel = {
          type: 'jsonmsg',
          mtype: 'reply',
          id: uuid.v4(),
          expires: Math.floor(Date.now() / 1000) + 60 * 5,
          replyid: msg.id,
          code: 404,
        };
        this.txData(this.primarygid, "device", datamodel, "jsondata", "", null, callback);
        return;
      } else if (target == "0.0.0.0") {
        listip = [];
        for (let h in this.hostManager.hosts.all) {
          let _host = this.hostManager.hosts.all[h];
          hosts.push(_host);
          listip.push(_host.o.ipv4Addr);
          if (_host.ipv6Addr && _host.ipv6Addr.length > 0) {
            for (let p in _host['ipv6Addr']) {
              listip.push(_host['ipv6Addr'][p]);
            }<commit_msg>bug fix<commit_after>#!/usr/bin/env node
/*    Copyright 2016 Firewalla LLC / Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

'use strict'

process.title = "FireApi";
let log = require('../net2/logger.js')(__filename, "info");

const util = require('util');
let fs = require('fs');

const ControllerBot = require('../lib/ControllerBot.js');

const sem = require('../sensor/SensorEventManager.js').getInstance();

let HostManager = require('../net2/HostManager.js');
let SysManager = require('../net2/SysManager.js');
let FlowManager = require('../net2/FlowManager.js');
let flowManager = new FlowManager('info');
let AlarmManager = require('../net2/AlarmManager.js');
let alarmManager = new AlarmManager('info');
let sysManager = new SysManager();
let VpnManager = require("../vpn/VpnManager.js");
let vpnManager = new VpnManager('info');
let IntelManager = require('../net2/IntelManager.js');
let intelManager = new IntelManager('debug');

let DeviceMgmtTool = require('../util/DeviceMgmtTool');

const Promise = require('bluebird');

const SysTool = require('../net2/SysTool.js')
const sysTool = new SysTool()

const flowUtil = require('../net2/FlowUtil');

const iptool = require('ip')

let redis = require('redis');
let rclient = redis.createClient();
var sclient = redis.createClient();
sclient.setMaxListeners(0);

Promise.promisifyAll(redis.RedisClient.prototype);

const exec = require('child-process-promise').exec

let AM2 = require('../alarm/AlarmManager2.js');
let am2 = new AM2();

let EM = require('../alarm/ExceptionManager.js');
let em = new EM();

let PM2 = require('../alarm/PolicyManager2.js');
let pm2 = new PM2();

let SSH = require('../extension/ssh/ssh.js');
let ssh = new SSH('info');

let country = require('../extension/country/country.js');

let builder = require('botbuilder');
let uuid = require('uuid');

let async2 = require('async');

let async = require('asyncawait/async');
let await = require('asyncawait/await');

let NM = require('../ui/NotifyManager.js');
let nm = new NM();

let FRP = require('../extension/frp/frp.js')
let frp = new FRP();

let f = require('../net2/Firewalla.js');

let flowTool = require('../net2/FlowTool')();

let i18n = require('../util/i18n');

let NetBotTool = require('../net2/NetBotTool');
let netBotTool = new NetBotTool();

let HostTool = require('../net2/HostTool');
let hostTool = new HostTool();

let appTool = require('../net2/AppTool')();

let spooferManager = require('../net2/SpooferManager.js')

const extMgr = require('../sensor/ExtensionManager')

class netBot extends ControllerBot {

  _block2(ip, dst, cron, timezone, duration, callback) {
    let value = {
      id: "0",
      cron: cron,
      dst: dst,
      timezone: timezone,
      duration: duration
    }
    this._block(ip, 'block', value, callback);
  }

  _ignore(ip, reason, callback) {
    this.hostManager.ignoreIP(ip, reason, callback);
  }

  _unignore(ip, callback) {
    this.hostManager.unignoreIP(ip, callback);
  }

  _block(ip, blocktype, value, callback) {
    log.info("_block", ip, blocktype, value);
    if (ip === "0.0.0.0") {
      this.hostManager.loadPolicy((err, data) => {
        this.hostManager.setPolicy(blocktype, value, (err, data) => {
          if (err == null) {
            if (callback != null)
              callback(null, "Success");
          } else {
            if (callback != null)
              callback(err, "Unable to block ip " + ip);
          }
        });
      });
    } else {
      this.hostManager.getHost(ip, (err, host) => {
        if (host != null) {
          host.loadPolicy((err, data) => {
            if (err == null) {
              host.setPolicy(blocktype, value, (err, data) => {
                if (err == null) {
                  if (callback != null)
                  //   this.tx(this.primarygid, "Success:"+ip,"hosts summary");
                    callback(null, "Success:" + ip);
                } else {
                  if (callback != null)
                  // this.tx(this.primarygid, "Unable to block ip "+ip,"hosts summary");
                    callback(err, "Unable to block ip " + ip)

                }
              });
            } else {
              if (callback != null)
              //this.tx(this.primarygid, "Unable to block ip "+ip,"hosts summary");
                callback("error", "Unable to block ip " + ip);
            }
          });
        } else {
          if (callback != null)
          //this.tx(this.primarygid, "host not found","hosts summary");
            callback("error", "Host not found");
        }
      });
    }
  }

  _family(ip, value, callback) {
    if (ip === "0.0.0.0") {
      this.hostManager.loadPolicy((err, data) => {
        this.hostManager.setPolicy("family", value, (err, data) => {
          if (err == null) {
            if (callback != null)
              callback(null, "Success");
          } else {
            if (callback != null)
              callback(err, "Unable to block ip " + ip);
          }
        });
      });
    } else {
      this.hostManager.getHost(ip, (err, host) => {
        if (host != null) {
          host.loadPolicy((err, data) => {
            if (err == null) {
              host.setPolicy("family", value, (err, data) => {
                if (err == null) {
                  if (callback != null)
                    callback(null, "Success:" + ip);
                } else {
                  if (callback != null)
                    callback(err, "Unable to block ip " + ip);
                }
              });
            } else {
              if (callback != null)
                callback("error", "Unable to block ip " + ip);
            }
          });
        } else {
          if (callback != null)
            callback("error", "host not found");
        }
      });
    }
  }

  _adblock(ip, value, callback) {
    if (ip === "0.0.0.0") {
      this.hostManager.loadPolicy((err, data) => {
        this.hostManager.setPolicy("adblock", value, (err, data) => {
          if (err == null) {
            if (callback != null)
              callback(null, "Success");
          } else {
            if (callback != null)
              callback(err, "Unable to block ip " + ip);
          }
        });
      });
    } else {
      this.hostManager.getHost(ip, (err, host) => {
        if (host != null) {
          host.loadPolicy((err, data) => {
            if (err == null) {
              host.setPolicy("adblock", value, (err, data) => {
                if (err == null) {
                  if (callback != null)
                    callback(null, "Success:" + ip);
                } else {
                  if (callback != null)
                    callback(err, "Unable to block ip " + ip);
                }
              });
            } else {
              if (callback != null)
                callback("error", "Unable to block ip " + ip);
            }
          });
        } else {
          if (callback != null)
            callback("error", "host not found");
        }
      });
    }
  }

  _vpn(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("vpn", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to block ip " + ip);
        }
      });
    });
  }

  _shadowsocks(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("shadowsocks", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to apply config on shadowsocks: " + value);
        }
      });
    });
  }

  _scisurf(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("scisurf", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to apply config on scisurf: " + value);
        }
      });
    });
  }

  _vulScan(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("vulScan", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to apply config on vulScan: " + value);
        }
      });
    });
  }

  _dnsmasq(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("dnsmasq", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to apply config on dnsmasq: " + value);
        }
      });
    });
  }

  _externalAccess(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("externalAccess", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to apply config on externalAccess: " + value);
        }
      });
    });
  }

  _ssh(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("ssh", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to ssh " + ip);
        }
      });
    });
  }

  /*
   *
   *   {
   *      state: BOOL;  overall notification
   *      ALARM_XXX: standard alarm definition
   *      ALARM_BEHAVIOR: may be mapped to other alarms
   *   }
   */
  _notify(ip, value, callback) {
    this.hostManager.loadPolicy((err, data) => {
      this.hostManager.setPolicy("notify", value, (err, data) => {
        if (err == null) {
          if (callback != null)
            callback(null, "Success");
        } else {
          if (callback != null)
            callback(err, "Unable to setNotify " + ip);
        }
        log.info("Notification Set",value," CurrentPolicy:", JSON.stringify(this.hostManager.policy.notify),{});
        nm.loadConfig();
      });
    });
  }

  constructor(config, fullConfig, eptcloud, groups, gid, debug, apiMode) {
    super(config, fullConfig, eptcloud, groups, gid, debug, apiMode);
    this.bot = new builder.TextBot();
    //      this.dialog = new builder.LuisDialog(config.dialog.api);
    this.dialog = new builder.CommandDialog();
    this.bot.add('/', this.dialog);
    let self = this;
    this.compress = true;
    this.scanning = false;

    this.sensorConfig = config.controller.sensor;
    //flow.summaryhours
    // sysManager.setConfig(this.sensorConfig);
    sysManager.update((err, data) => {
    });

    setInterval(() => {
      sysManager.update((err, data) => {
      });
    }, 1000 * 60 * 60 * 10);

    setInterval(() => {
      try {
        if (global.gc) {
          global.gc();
        }
      } catch (e) {
      }
    }, 1000 * 60);

    setTimeout(() => {
      setInterval(() => {
//          this.refreshCache(); // keep cache refreshed every 50 seconds so that app will load data fast
      }, 50 * 1000);
    }, 30 * 1000)

    this.hostManager = new HostManager("cli", 'client', 'debug');
    this.hostManager.loadPolicy((err, data) => {});  //load policy
 
    // no subscription for api mode
    if (apiMode) {
      log.info("Skipping event subscription during API mode.");
      return;
    }

    let c = require('../net2/MessageBus.js');
    this.subscriber = new c('debug');

    this.subscriber.subscribe("DiscoveryEvent", "DiscoveryStart", null, (channel, type, ip, msg) => {
      //this.tx(this.primarygid, "Discovery started","message");
    });
    this.subscriber.subscribe("DiscoveryEvent", "Host:Found", null, (channel, type, ip, o) => {
      log.info("Found new host ", channel, type, ip);
      if (o) {
        let name = o.ipv4Addr;
        if (o.name != null) {
          name = o.name + " (" + o.ipv4Addr + ")";
        } else if (o.macVendor != null) {
          name = "(?)" + o.macVendor + " (" + o.ipv4Addr + ")";
        }
        this.tx2(this.primarygid, "New host found in network: " + name, "Found new host " + name, {uid: o.ipv4Addr});
      }
    });
    this.subscriber.subscribe("MonitorEvent", "Monitor:Flow:Out", null, (channel, type, ip, msg) => {
      let m = null;
      let n = null;
      log.info("Monitor:Flow:Out", channel, ip, msg, "=====");
      if (ip && msg) {
        if (msg['txRatioRanked'] && msg['txRatioRanked'].length > 0) {
          let flow = msg['txRatioRanked'][0];
          if (flow.rank > 0) {
            return;
          }
          m = "Warning: \n\n" + flowManager.toStringShortShort2(msg['txRatioRanked'][0], msg.direction, 'txdata') + "\n";
          n = flowManager.toStringShortShort2(msg['txRatioRanked'][0], msg.direction);
        }
      }
      if (m) {
        log.info("MonitorEvent:Flow:Out", m, msg);
        if (nm.canNotify() == true) {
          this.tx2(this.primarygid, m, n, {id: msg.id});
        }
      }
    });
    this.subscriber.subscribe("MonitorEvent", "Monitor:Flow:In", null, (channel, type, ip, msg) => {
      /*
       let m = null;
       let n = null;
       log.info("Monitor:Flow:In", channel, ip, msg, "=====");
       if (ip && msg) {
       if (msg['txRatioRanked'] && msg['txRatioRanked'].length > 0) {
       let flow = msg['txRatioRanked'][0];
       if (flow.rank > 0) {
       return;
       }
       m = "Warning: \n\n" + flowManager.toStringShortShort2(msg['txRatioRanked'][0], msg.direction, 'txdata') + "\n";
       n = flowManager.toStringShortShort2(msg['txRatioRanked'][0], msg.direction);
       }
       }
       if (m)
       this.tx2(this.primarygid, m, n, {id:msg.id});
       */
    });


    this.subscriber.subscribe("ALARM", "ALARM:CREATED", null, (channel, type, ip, msg) => {
      if (msg) {
        let notifMsg = msg.notif;
        let aid = msg.aid;
        if (notifMsg) {
          log.info("Sending notification: " + JSON.stringify(msg));
          if (this.hostManager.policy && this.hostManager.policy["notify"]) {
               if (this.hostManager.policy['notify']['state']==false) {
                   log.info("ALARM_NOTIFY_GLOBALL_BLOCKED", msg);
                   return;
               }
               if (msg.alarmType) {
                   let alarmType = msg.alarmType;
                   if (msg.alarmType  === "ALARM_LARGE_UPDATE") {
                       alarmType = "ALARM_BEHAVIOR";
                   }
                   if (this.hostManager.policy["notify"][alarmType] === false) {
                       log.info("ALARM_NOTIFY_BLOCKED", msg);
                       return;
                   }
               }
          }

          log.info("ALARM_NOTIFY_PASSED");

          notifMsg = {
            title: i18n.__("SECURITY_ALERT"),
            body: notifMsg
          }

          let data = {
            gid: this.primarygid,
          };

          if (msg.aid) {
            data.aid = msg.aid;
          }

          if (msg.alarmID) {
            data.alarmID = msg.alarmID;
          }

          switch(msg.alarmNotifType) {
            case "security":
              notifMsg.title = i18n.__("SECURITY_ALERT");
              break;
            case "activity":
              notifMsg.title = i18n.__("ACTIVITY_ALERT");
              break;
            default:
              break;
          }

          if (msg.autoblock) {
            data.category = "com.firewalla.category.autoblockalarm";
          } else {
            data.category = "com.firewalla.category.alarm";
          }

          // check if device name should be included, sometimes it is helpful if multiple devices are bound to one app
          async(() => {
            let flag = await (rclient.hgetAsync("sys:config", "includeNameInNotification"))
            if(flag) {
              notifMsg.body = `[${this.getDeviceName()}] ${notifMsg.body}`
            }
            this.tx2(this.primarygid, "test", notifMsg, data);            
          })()


        }
      }
    });

    setTimeout(() => {
      this.scanStart();
      async(() => {
        let branchChanged = await (sysManager.isBranchJustChanged())
        if(branchChanged) {
          let branch = null
          
          switch(branchChanged) {
          case "1":
            branch = "back to stable version"
            break;
          case "2":
            branch = "to pre_release version"
            break;
          case "3":
            branch = "to development version"
            break;
          default:
            // do nothing, should not happen here
            break;
          }

          if(branch) {
            let msg = `Device '${this.getDeviceName()}' has switched ${branch} ${sysManager.version()} successfully`
            log.info(msg)
            this.tx(this.primarygid, "200", msg)
            sysManager.clearBranchChangeFlag()            
          }

        } else {
          if (sysManager.systemRebootedDueToIssue(true) == false) {
            if (nm.canNotify() == true) {
              this.tx(this.primarygid, "200", "🔥 Firewalla Device '" + this.getDeviceName() + "' Awakens!");
            }
          }
        }
        
      })()
      this.setupDialog();
    }, 2000);

    this.hostManager.on("Scan:Done", (channel, type, ip, obj) => {
      if (type == "Scan:Done") {
        this.scanning = false;
        for (let h in this.hosts) {
          //this.hosts[h].clean();
        }
        this.scanStart();
      }
    });
    this.hostManager.on("Scan:Start", (channel, type, ip, obj) => {
      if (type == "Scan:Start") {
        this.scanning = true;
      }
    });

    sclient.on("message", (channel, msg)=> {
       log.info("Msg",channel,msg);
       switch(channel) {
         case "System:Upgrade:Hard":
             if (msg) {
                let notifyMsg = {
                   title: "Upgrade Needed",
                   body: "Firewalla new version "+msg+" is avaliable, please open firewalla app then tap on settings->upgrade.",
                   } 
                let data = {
                   gid: this.primarygid,
                };
                this.tx2(this.primarygid, "", notifyMsg, data);
             }
             break;
         case "System:Upgrade:Soft":
             if (msg) {
                let notifyMsg = {
                  title: `Firewalla is upgraded to ${msg}`,
                  body: ""
                }
                let data = {
                  gid: this.primarygid,
                };
                this.tx2(this.primarygid, "", notifyMsg, data);
             }
             break;
       }
    });
    sclient.subscribe("System:Upgrade:Hard");
    sclient.subscribe("System:Upgrade:Soft");


  }

  boneMsgHandler(msg) {
      if (msg.type == "MSG" && msg.title) {
          let notifyMsg = {
             title: msg.title,
             body: msg.body 
          } 
          let data = {
             gid: this.primarygid,
             data: msg.data
          };
          this.tx2(this.primarygid, "", notifyMsg, data);
      }
  }

  scanStart(callback) {
    this.hostManager.getHosts((err, result) => {
      let listip = [];
      this.hosts = result;
      for (let i in result) {
//        log.info(result[i].toShortString());
        result[i].on("Notice:Detected", (channel, type, ip, obj) => {
          log.info("Found new notice", type, ip);
          if ((obj.note == "Scan::Port_Scan" || obj.note == "Scan::Address_Scan") && this.scanning == false) {
            let msg = result[i].name() + ": " + obj.msg;
            if (nm.canNotify() == true) {
              this.tx(this.primarygid, msg, obj.msg);
            }
          } else if ((obj.note == "Scan::Port_Scan" || obj.note == "Scan::Address_Scan") && this.scanning == true) {
            log.info("Netbot:Notice:Skip due to scanning", obj);
          } else {
            let msg = result[i].name() + ": " + obj.msg;
            if (nm.canNotify() == true) {
              this.tx(this.primarygid, msg, obj.msg);
            }
          }
        });
      }
      if (callback)
        callback(null, null);

    });

  }

  setHandler(gid, msg, callback) {
    // mtype: set
    // target = "ip address" 0.0.0.0 is self
    // data.item = policy
    // data.value = {'block':1},
    //
    //       log.info("Set: ",gid,msg);

    // invalidate cache
    this.invalidateCache();


    switch (msg.data.item) {
      case "policy":
        async2.eachLimit(Object.keys(msg.data.value), 1, (o, cb) => {
          switch (o) {
            case "monitor":
              this._block(msg.target, "monitor", msg.data.value.monitor, (err, obj) => {
                cb(err);
              });
              break;
            case "blockin":
              this._block(msg.target, "blockin", msg.data.value.blockin, (err, obj) => {
                cb(err);
              });
              break;
            case "acl":
              this._block(msg.target, "acl", msg.data.value.acl, (err, obj) => {
                cb(err);
              });
              break;
            case "family":
              this._family(msg.target, msg.data.value.family, (err, obj) => {
                cb(err);
              });
              break;
            case "adblock":
              this._adblock(msg.target, msg.data.value.adblock, (err, obj) => {
                cb(err);
              });
              break;
            case "vpn":
              this._vpn(msg.target, msg.data.value.vpn, (err, obj) => {
                cb(err);
              });
              break;
            case "shadowsocks":
              this._shadowsocks(msg.target, msg.data.value.shadowsocks, (err, obj) => {
                cb(err);
              });
              break;
            case "scisurf":
              this._scisurf(msg.target, msg.data.value.scisurf, (err, obj) => {
                cb(err);
              });
              break;
            case "vulScan":
              this._vulScan(msg.target, msg.data.value.vulScan, (err, obj) => {
                cb(err);
              });
              break;
            case "dnsmasq":
              this._dnsmasq(msg.target, msg.data.value.dnsmasq, (err, obj) => {
                cb(err);
              });
              break;
            case "externalAccess":
              this._externalAccess(msg.target, msg.data.value.externalAccess, (err, obj) => {
                cb(err);
              });
              break;
            case "ssh":
              this._ssh(msg.target, msg.data.value.ssh, (err, obj) => {
                cb(err);
              });
              break;
            case "ignore":
              this._ignore(msg.target, msg.data.value.ignore.reason, (err, obj) => {
                cb(err);
              });
              break;
            case "unignore":
              this._unignore(msg.target, (err, obj) => {
                cb(err);
              });
              break;
            case "notify":
              this._notify(msg.target, msg.data.value.notify, (err, obj) => {
                cb(err);
              });
              break;
          default:
            let target = msg.target
            let policyData = msg.data.value[o]

            //            if(extMgr.hasExtension(o)) {
              this.hostManager.loadPolicy((err, data) => {
                this.hostManager.setPolicy(o,
                                           policyData,
                                           (err, data) => {
                  cb(err)
                })
              })
            // } else {
            //   cb(null)
            // }
            break
          }
        }, (err) => {
          let reply = {
            type: 'jsonmsg',
            mtype: 'policy',
            id: uuid.v4(),
            expires: Math.floor(Date.now() / 1000) + 60 * 5,
            replyid: msg.id,
          };
          reply.code = 200;
          reply.data = msg.data.value;
          log.info("Repling ", reply.code, reply.data);
          this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);

        });
        break;
      case "host":
        //data.item = "host" test
        //data.value = "{ name: " "}"
        let data = msg.data;
        log.info("Setting Host", msg);
        let reply = {
          type: 'jsonmsg',
          mtype: 'init',
          id: uuid.v4(),
          expires: Math.floor(Date.now() / 1000) + 60 * 5,
          replyid: msg.id,
        };
        reply.code = 200;

        this.hostManager.getHost(msg.target, (err, host) => {
          if (host == null) {
            log.info("Host not found");
            reply.code = 404;
            this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);
            return;
          }

          if (data.value.name == host.o.name) {
            log.info("Host not changed", data.value.name, host.o.name);
            reply.code = 200;
            this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);
            return;
          }

          host.o.name = data.value.name;
          log.info("Changing names", host.o.name);
          host.save(null, (err) => {
            if (err) {
              reply.code = 500;
              this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);
            } else {
              reply.code = 200;
              reply.data = msg.data.value;
              this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);
            }
          });
        });
        break;
      case "intel":
        // intel actions
        //   - ignore / unignore
        //   - report
        //   - block / unblockj
        intelManager.action(msg.target, msg.data.value.action, (err) => {
          let reply = {
            type: 'jsonmsg',
            mtype: 'init',
            id: uuid.v4(),
            expires: Math.floor(Date.now() / 1000) + 60 * 5,
            replyid: msg.id,
          };
          reply.code = 200;
          this.txData(this.primarygid, "", reply, "jsondata", "", null, callback);
        });
        break;
      case "scisurfconfig":
        let v = msg.data.value;

        // TODO validate input ??
        if (v.from && v.from === "firewalla") {
          let scisurf = require('../extension/ss_client/ss_client.js');
          scisurf.saveConfig(v, (err) => {
            this.simpleTxData(msg, {}, err, callback);
          });
        } else {
          this.simpleTxData(msg, {}, new Error("Invalid config"), callback);
        }

        break;
      case "language":
        let v2 = msg.data.value;

        // TODO validate input?
        if (v2.language) {
          sysManager.setLanguage(v2.language, (err) => {
            this.simpleTxData(msg, {}, err, callback);
          });
        }
        break;
      case "timezone":
        let v3 = msg.data.value;

        if (v3.timezone) {
          sysManager.setTimezone(v3.timezone, (err) => {
            this.simpleTxData(msg, {}, err, callback);
          });
        }
      break;
    case "includeNameInNotification":
        let v33 = msg.data.value;

      if (v3.includeNameInNotification) {
        async(() => {
          await (rclient.hsetAsync("sys:config", "includeNameInNotification", "1"))
          this.simpleTxData(msg, {}, null, callback)
        })().catch((err) => {
          this.simpleTxData(msg, {}, err, callback)
        })
      }
      break;
    case "mode":
      let v4 = msg.data.value;
      let err = null;
      
      if (v4.mode) {
        async(() => {
          let mode = require('../net2/Mode.js')
          let curMode = await (mode.getSetupMode())        
          if(v4.mode === curMode) {
            this.simpleTxData(msg, {}, err, callback);
            return
          }
          
          let modeManager = require('../net2/ModeManager.js');
          switch (v4.mode) {
          case "spoof":
          case "autoSpoof":
            modeManager.setAutoSpoofAndPublish()
            break;
          case "manualSpoof":
            modeManager.setManualSpoofAndPublish()
            break;
          case "dhcp":
            modeManager.setDHCPAndPublish()
            break;
          case "none":
            modeManager.setNoneAndPublish()
            break;
          default:
            log.error("unsupported mode: " + v4.mode);
            err = new Error("unsupport mode: " + v4.mode);
            break;
          }
          this.simpleTxData(msg, {}, err, callback);
        })().catch((err) => {
          this.simpleTxData(msg, {}, err, callback);
        })
      }
      break;
    default:
        this.simpleTxData(msg, null, new Error("Unsupported action"), callback);
        break;
    }
  }


  getAllIPForHost(ip, callback) {
    let listip = [];
    this.hostManager.getHost(ip, (err, host) => {
      if (host != null) {
        listip.push(host.o.ipv4Addr);
        if (host.ipv6Addr && host.ipv6Addr.length > 0) {
          for (let j in host['ipv6Addr']) {
            listip.push(host['ipv6Addr'][j]);
          }
        }
      }
      callback(err, listip);
    });
  }




  getHandler(gid, msg, appInfo, callback) {

    // backward compatible
    if(typeof appInfo === 'function') {
      callback = appInfo;
      appInfo = undefined;
    }

    // mtype: get
    // target = ip address
    // data.item = [app, alarms, host]

    switch (msg.data.item) {
      case "host":
        if (msg.target) {
          let useNewDeviceHandler = appTool.isAppReadyForNewDeviceHandler(appInfo);
          if(useNewDeviceHandler) {
            let ip = msg.target;
            log.info("Loading device info in a new way:", ip);
            this.newDeviceHandler(msg, ip)
            .then((json) => {
              this.simpleTxData(msg, json, null, callback);
            })
            .catch((err) => {
              this.simpleTxData(msg, null, err, callback);
            })
          } else {
            log.info("Using the legacy way to get device info:", msg.target, {});
            this.getAllIPForHost(msg.target, (err, ips) => {
              this.deviceHandler(msg, gid, msg.target, ips, callback);
            });
          }
        }
        break;
      case "vpn":
      case "vpnreset":
        let regenerate = true;
        if (msg.data.item === "vpnreset") {
          regenerate = true;
        }

        this.hostManager.loadPolicy(() => {
          vpnManager.getOvpnFile("fishboneVPN1", null, regenerate, (err, ovpnfile, password) => {
            let datamodel = {
              type: 'jsonmsg',
              mtype: 'reply',
              id: uuid.v4(),
              expires: Math.floor(Date.now() / 1000) + 60 * 5,
              replyid: msg.id,
              code: 404,
            };
            if (err == null) {
              datamodel.code = 200;
              datamodel.data = {
                ovpnfile: ovpnfile,
                password: password,
                portmapped: this.hostManager.policy['vpnPortmapped']
              }
            }
            this.txData(this.primarygid, "device", datamodel, "jsondata", "", null, callback);
          });
        });
        break;
      case "shadowsocks":
      case "shadowsocksResetConfig":
        let shadowsocks = require('../extension/shadowsocks/shadowsocks.js');
        let ss = new shadowsocks('info');

        if (msg.data.item === "shadowsocksResetConfig") {
          ss.refreshConfig();
        }

        let config = ss.readConfig();
        let datamodel = {
          type: 'jsonmsg',
          mtype: 'reply',
          id: uuid.v4(),
          expires: Math.floor(Date.now() / 1000) + 60 * 5,
          replyid: msg.id,
          code: 200,
          data: {
            config: config
          }
        };
        this.txData(this.primarygid, "device", datamodel, "jsondata", "", null, callback);
        break;
      case "sshPrivateKey":

        ssh.getPrivateKey((err, data) => {
          if (err) {
            log.error("Got error when loading ssh private key: " + err);
            data = "";
          }

          let datamodel = {
            type: 'jsonmsg',
            mtype: 'reply',
            id: uuid.v4(),
            expires: Math.floor(Date.now() / 1000) + 60 * 5,
            replyid: msg.id,
            code: 200,
            data: {
              key: data
            }
          };
          this.txData(this.primarygid, "device", datamodel, "jsondata", "", null, callback);
        });
        break;
      case "sshRecentPassword":
        ssh.getPassword((err, password) => {

          let data = "";

          if (err) {
            log.error("Got error when reading password: " + err);
            this.simpleTxData(msg, {}, err, callback);
          } else {
            this.simpleTxData(msg, {password: password}, err, callback);
          }
        });
        break;
      case "sysInfo":
        let si = require('../extension/sysinfo/SysInfo.js');
        this.simpleTxData(msg, si.getSysInfo(), null, callback);
        break;
      case "logFiles":
        let si2 = require('../extension/sysinfo/SysInfo.js');
        si2.getRecentLogs((err, results) => {
          this.simpleTxData(msg, results, null, callback);
        });
        break;
      case "scisurfconfig":
        let ssc = require('../extension/ss_client/ss_client.js');
        ssc.loadConfig((err, result) => {
          this.simpleTxData(msg, result || {}, err, callback);
        });
        break;
      case "language":
        this.simpleTxData(msg, {language: sysManager.language}, null, callback);
        break;
      case "timezone":
        this.simpleTxData(msg, {timezone: sysManager.timezone}, null, callback);
        break;
      case "alarms":
        am2.loadActiveAlarms((err, alarms) => {
          this.simpleTxData(msg, {alarms: alarms, count: alarms.length}, err, callback);
        });
        break;
      case "alarm":
        let alarmID = msg.data.value.alarmID;
        am2.getAlarm(alarmID)
          .then((alarm) => this.simpleTxData(msg, alarm, null, callback))
          .catch((err) => this.simpleTxData(msg, null, err, callback));
        break;
      case "exceptions":
        em.loadExceptions((err, exceptions) => {
          this.simpleTxData(msg, {exceptions: exceptions, count: exceptions.length}, err, callback);
        });
        break;
    case "frpConfig":
      let _config = frp.getConfig()
      if(_config.started) {
        let getPasswordAsync = Promise.promisify(ssh.getPassword)
        getPasswordAsync().then((password) => {
          _config.password = password
          this.simpleTxData(msg, _config, null, callback);
        }).catch((err) => {
          this.simpleTxData(msg, null, err, callback);
        })
      } else {
        this.simpleTxData(msg, _config, null, callback);
      }
      break;
    default:
        this.simpleTxData(msg, null, new Error("unsupported action"), callback);
    }
  }

  validateFlowIntel(json) {
    return async(() => {
      // await (bone.flowgraphAsync(...))
      let flows = json.flows

      let hashCache = {}


      let appFlows = flows.appDetails
      let categoryFlows = flows.categoryDetails

      if(Object.keys(appFlows).length > 0 ||
         Object.keys(categoryFlows).length > 0) {
        flowUtil.hashIntelFlows(appFlows, hashCache)
        flowUtil.hashIntelFlows(categoryFlows, hashCache)
        
        let data = await (bone.flowgraphAsync('summarizeApp', appFlows),
                          bone.flowgraphAsync('summarizeActivity', categoryFlows))

        let unhashedData = flowUtil.unhashIntelFlows(data[0], hashCache)
        let unhashedData2 = flowUtil.unhashIntelFlows(data[1], hashCache)
        
        flows.appDetails = unhashedData
        flows.categoryDetails = unhashedData2
      }
    })()
  }
  
  systemFlowHandler(msg) {
    log.info("Getting flow info of the entire network");

    let begin = msg.data && msg.data.start;
    //let end = msg.data && msg.data.end;
    let end = begin && (begin + 3600);

    if(!begin || !end) {
      return Promise.reject(new Error("Require begin and error when calling systemFlowHandler"));
    }

    log.info("FROM: ", new Date(begin * 1000).toLocaleTimeString());
    log.info("TO: ", new Date(end * 1000).toLocaleTimeString());

    return async(() => {
      let jsonobj = {};
      let options = {
        begin: begin,
        end: end
      }
      await (flowTool.prepareRecentFlows(jsonobj, options))
      await (netBotTool.prepareTopUploadFlows(jsonobj, options))
      await (netBotTool.prepareTopDownloadFlows(jsonobj, options))

      await (netBotTool.prepareDetailedAppFlowsFromCache(jsonobj, options))
      if(!jsonobj.flows['appDetails']) { // fallback to old way
        await (netBotTool.prepareDetailedAppFlows(jsonobj, options))
        await (this.validateFlowIntel(jsonobj))
      }

      await (netBotTool.prepareDetailedCategoryFlowsFromCache(jsonobj, options))
      if(!jsonobj.flows['categoryDetails']) { // fallback to old model
        await (netBotTool.prepareDetailedCategoryFlows(jsonobj, options))
        await (this.validateFlowIntel(jsonobj))
      }

      return jsonobj;
    })();
  }

  newDeviceHandler(msg, ip) {
    log.info("Getting info on device", ip, {});

    return async(() => {
      if(ip === '0.0.0.0') {
        return this.systemFlowHandler(msg);
      }

      let begin = msg.data && msg.data.start;
      let end = (msg.data && msg.data.end) || begin + 3600 * 24;

      let options = {}
      if(begin && end) {
        options.begin = begin
        options.end = end
      }

      if(msg.data.hourblock != "1" &&
         msg.data.hourblock != "0" ) { // 0 => now, 1 => hour stats, other => overall stats (last 24 hours)
        options.queryall = true
      }
      
      let host = await (this.hostManager.getHostAsync(ip));
      if(!host || !host.o.mac) {
        let error = new Error("Invalide Host");
        error.code = 404;
        return Promise.reject(error);
      }

      let mac = host.o.mac;

      // load 24 hours download/upload trend
      await (flowManager.getStats2(host));

      let jsonobj = {};
      if (host) {
        jsonobj = host.toJson();

        await (flowTool.prepareRecentFlowsForHost(jsonobj, mac, options));
        await (netBotTool.prepareTopUploadFlowsForHost(jsonobj, mac, options));
        await (netBotTool.prepareTopDownloadFlowsForHost(jsonobj, mac, options));

        await (netBotTool.prepareAppActivityFlowsForHost(jsonobj, mac, options))
        await (netBotTool.prepareCategoryActivityFlowsForHost(jsonobj, mac, options))

        await (netBotTool.prepareDetailedAppFlowsForHostFromCache(jsonobj, mac, options))
        if(!jsonobj.flows["appDetails"]) {
          log.warn("Fell back to legacy mode on app details:", mac, options, {})
          await (netBotTool.prepareAppActivityFlowsForHost(jsonobj, mac, options))
          await (this.validateFlowIntel(jsonobj))
        }

        await (netBotTool.prepareDetailedCategoryFlowsForHostFromCache(jsonobj, mac, options))
        if(!jsonobj.flows["categoryDetails"]) {
          log.warn("Fell back to legacy mode on category details:", mac, options, {})
          await (netBotTool.prepareCategoryActivityFlowsForHost(jsonobj, mac, options))
          await (this.validateFlowIntel(jsonobj))
        }
      }

      return jsonobj;
    })();
  }

  deviceHandler(msg, gid, target, listip, callback) {
    log.info("Getting Devices", gid, target, listip);
    let hosts = [];
    this.hostManager.getHost(target, (err, host) => {
      if (host == null && target != "0.0.0.0") {
        let datamodel = {
          type: 'jsonmsg',
          mtype: 'reply',
          id: uuid.v4(),
          expires: Math.floor(Date.now() / 1000) + 60 * 5,
          replyid: msg.id,
          code: 404,
        };
        this.txData(this.primarygid, "device", datamodel, "jsondata", "", null, callback);
        return;
      } else if (target == "0.0.0.0") {
        listip = [];
        for (let h in this.hostManager.hosts.all) {
          let _host = this.hostManager.hosts.all[h];
          hosts.push(_host);
          listip.push(_host.o.ipv4Addr);
          if (_host.ipv6Addr && _host.ipv6Addr.length > 0) {
            for (let p in _host['ipv6Addr']) {
              listip.push(_host['ipv6Addr'][p]);
            }
{'fraction_non_alphanumeric': 0.14730570698055864, 'fraction_numerical': 0.007332722273143905, 'mean_word_length': 1.7081128747795415, 'pattern_counts': {'":': 106, '<': 5, '<?xml version=': 0, '>': 229, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 14, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4699830', 'n_tokens_mistral': 25308, 'n_tokens_neox': 24525, 'n_words': 5327}