<commit_before>/*    Copyright 2016 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';
let log = require('./logger.js')(__filename);

var iptool = require('ip');
var os = require('os');
var network = require('network');
var instances = {};

const rclient = require('../util/redis_manager.js').getRedisClient()
const sclient = require('../util/redis_manager.js').getSubscriptionClient()

const exec = require('child-process-promise').exec

let Promise = require('bluebird');

const timeSeries = require('../util/TimeSeries.js').getTimeSeries()
const getHitsAsync = Promise.promisify(timeSeries.getHits).bind(timeSeries)

var Spoofer = require('./Spoofer.js');
var spoofer = null;
var SysManager = require('./SysManager.js');
var sysManager = new SysManager('info');
var DNSManager = require('./DNSManager.js');
var dnsManager = new DNSManager('error');
var FlowManager = require('./FlowManager.js');
var flowManager = new FlowManager('debug');

const DNSMASQ = require('../extension/dnsmasq/dnsmasq.js');

const FRPManager = require('../extension/frp/FRPManager.js')
const fm = new FRPManager()
const frp = fm.getSupportFRP()

const AlarmManager2 = require('../alarm/AlarmManager2.js');
const alarmManager2 = new AlarmManager2();

const PolicyManager2 = require('../alarm/PolicyManager2.js');
const policyManager2 = new PolicyManager2();
const pm2 = policyManager2

let ExceptionManager = require('../alarm/ExceptionManager.js');
let exceptionManager = new ExceptionManager();

let spooferManager = require('./SpooferManager.js')

let modeManager = require('./ModeManager.js');

let async = require('asyncawait/async');
let await = require('asyncawait/await');

let f = require('./Firewalla.js');

const getPreferredBName = require('../util/util.js').getPreferredBName

const license = require('../util/license.js')

var alarmManager = null;

var uuid = require('uuid');
var bone = require("../lib/Bone.js");

var utils = require('../lib/utils.js');

let fConfig = require('./config.js').getConfig();

const fc = require('./config.js')

var _async = require('async');

var MobileDetect = require('mobile-detect');

var flowUtil = require('../net2/FlowUtil.js');

let AppTool = require('./AppTool');
let appTool = new AppTool();

var linux = require('../util/linux.js');

const HostTool = require('../net2/HostTool.js')
const hostTool = new HostTool()

/* alarms:
    alarmtype:  intel/newhost/scan/log
    severityscore: out of 100
    alarmseverity: major minor
*/


class Host {
    constructor(obj, mgr, callback) {
      this.callbacks = {};
      this.o = obj;
      this.mgr = mgr;
      if (this.o.ipv4) {
        this.o.ipv4Addr = this.o.ipv4;
      }

      this._mark = false;
      this.parse();

      let c = require('./MessageBus.js');
      this.subscriber = new c('debug');

      if (this.mgr.type === 'server') {
        this.spoofing = false;
        sclient.on("message", (channel, message) => {
          this.processNotifications(channel, message);
        });

        if (obj != null) {
          this.subscribe(this.o.ipv4Addr, "Notice:Detected");
          this.subscribe(this.o.ipv4Addr, "Intel:Detected");
          this.subscribe(this.o.ipv4Addr, "HostPolicy:Changed");
        }
        this.spoofing = false;

        /*
         if (this.o.ipv6Addr) {
         this.o.ipv6Addr = JSON.parse(this.o.ipv6Addr);
         }
         */
        this.predictHostNameUsingUserAgent();

        this.loadPolicy(callback);
      }

      this.dnsmasq = new DNSMASQ();
    }

    update(obj) {
        this.o = obj;
        if (this.o.ipv4) {
            this.o.ipv4Addr = this.o.ipv4;
        }

        if(this.mgr.type === 'server') {
          if (obj != null) {
            this.subscribe(this.o.ipv4Addr, "Notice:Detected");
            this.subscribe(this.o.ipv4Addr, "Intel:Detected");
            this.subscribe(this.o.ipv4Addr, "HostPolicy:Changed");
          }
          this.predictHostNameUsingUserAgent();
          this.loadPolicy(null);
        }

        this.parse();
    }

/* example of ipv6Host
1) "mac"
2) "B8:53:AC:5F:99:51"
3) "firstFoundTimestamp"
4) "1511599097.786"
5) "lastActiveTimestamp"
6) "1511846844.798"

*/

    keepalive() {
        for (let i in this.ipv6Addr) {
            log.debug("keep alive ", this.mac,this.ipv6Addr[i]);
            linux.ping6(null,this.ipv6Addr[i]);
        }
        setTimeout(()=>{
            this.cleanV6();
        },1000*10);
    }

    cleanV6() {
        return async(()=> {
            if (this.ipv6Addr == null) {
                return;
            }

            let ts = (new Date())/1000;
            let lastActive = 0;
            let _ipv6Hosts = {};
            this._ipv6Hosts = {};

            for (let i in this.ipv6Addr) {
                let ip6 = this.ipv6Addr[i];
                let ip6Host = await(rclient.hgetallAsync("host:ip6:"+ip6));
                log.debug("HostManager:CleanV6:looking up v6",ip6,ip6Host)
                if (ip6Host != null) {
                    _ipv6Hosts[ip6] = ip6Host;
                    if (ip6Host.lastActiveTimestamp > lastActive) {
                        lastActive = ip6Host.lastActiveTimestamp;
                    }
                }
            }

            this.ipv6Addr = [];
            for (let ip6 in _ipv6Hosts) {
                let ip6Host = _ipv6Hosts[ip6];
                if (ip6Host.lastActiveTimestamp < lastActive - 60*30 || ip6Host.lastActiveTimestamp < ts-60*40) {
                    log.info("Host:"+this.mac+","+ts+","+ip6Host.lastActiveTimestamp+","+lastActive+" Remove Old Address "+ip6,JSON.stringify(ip6Host));
                } else {
                    this._ipv6Hosts[ip6] = ip6Host;
                    this.ipv6Addr.push(ip6);
                }
            }

            if (this.o.lastActiveTimestamp < lastActive) {
                this.o.lastActiveTimestamp = lastActive;
            }

            //await(this.saveAsync());
            log.debug("HostManager:CleanV6:", this.o.mac, JSON.stringify(this.ipv6Addr));
        })().catch((err) => {
            log.error("Got error when cleanV6", err, {})            
        });
    }

    predictHostNameUsingUserAgent() {
        if (this.hasBeenGivenName() == false) {
            rclient.smembers("host:user_agent_m:" + this.o.mac, (err, results) => {
                if (results != null && results.length > 0) {
                    let familydb = {};
                    let osdb = {};
                    let mobile = false;
                    let md_osdb = {};
                    let md_name = {};

                    for (let i in results) {
                        let r = JSON.parse(results[i]);
                        if (r.ua) {
                            let md = new MobileDetect(r.ua);
                            if (md == null) {
                                log.info("MD Null");
                                continue;
                            }
                            let name = null;
                            if (md.mobile()) {
                                mobile = true;
                                name = md.mobile();
                            }
                            let os = md.os();
                            if (os != null) {
                                if (md_osdb[os]) {
                                    md_osdb[os] += 1;
                                } else {
                                    md_osdb[os] = 1;
                                }
                            }
                            if (name != null) {
                                if (md_name[name]) {
                                    md_name[name] += 1;
                                } else {
                                    md_name[name] = 1;
                                }
                            }
                        }

                        /*
                        if (r!=null) {
                            if (r.family.indexOf("Other")==-1) {
                                if (familydb[r.family]) {
                                   familydb[r.family] += 1;
                                } else {
                                   familydb[r.family] = 1;
                                }
                                bestFamily = r.family;
                            } else if (r.os.indexOf("Other")==-1) {
                                bestOS = r.os;
                            }
                            break;
                        }
                        */
                    }
                    log.debug("Sorting", JSON.stringify(md_name), JSON.stringify(md_osdb))
                    let bestOS = null;
                    let bestName = null;
                    let osarray = [];
                    let namearray = [];
                    for (let i in md_osdb) {
                        osarray.push({
                            name: i,
                            rank: md_osdb[i]
                        });
                    }
                    for (let i in md_name) {
                        namearray.push({
                            name: i,
                            rank: md_name[i]
                        });
                    }
                    osarray.sort(function (a, b) {
                        return Number(b.rank) - Number(a.rank);
                    })
                    namearray.sort(function (a, b) {
                        return Number(b.rank) - Number(a.rank);
                    })
                    if (namearray.length > 0) {
                        this.o.ua_name = namearray[0].name;
                        this.predictedName = "(?)" + this.o.ua_name;

                        if (osarray.length > 0) {
                            this.o.ua_os_name = osarray[0].name;
                            this.predictedName += "/" + this.o.ua_os_name;
                        }
                        this.o.pname = this.predictedName;
                        log.debug(">>>>>>>>>>>> ", this.predictedName, JSON.stringify(this.o.ua_os_name));
                    }
                    if (mobile == true) {
                        this.o.deviceClass = "mobile";
                        this.save("deviceClass", null);
                    }
                    if (this.o.ua_os_name) {
                        this.save("ua_os_name", null);
                    }
                    if (this.o.ua_name) {
                        this.save("ua_name", null);
                    }
                    if (this.o.pname) {
                        this.save("pname", null);
                    }
                    //this.save("ua_name",null);

                    /*
                    if (bestFamily!=null) {
                        this.hostname = "(?)"+bestFamily;
                    } else if (bestOS!=null) {
                        this.hostname = "(?)"+bestOS;
                    }
                    log.info(this.o.name,this.hostname);
                    if (this.hostname!=null && this.o.name!=this.hostname) {
                        //this.o.name = this.hostname;
                        this.save("name",null);
                    }
                   */
                }
            });

            /*
                        rclient.smembers("host:user_agent:"+this.o.ipv4Addr,(err,results)=> {
                            if (results!=null && results.length>0) {
                                let bestFamily = null;
                                let bestOS = null;
                                for (let i in results) {
                                    let r = JSON.parse(results[i]);
                                    if (r!=null) {
                                        if (r.family.indexOf("Other")==-1) {
                                            bestFamily = r.family;
                                        } else if (r.os.indexOf("Other")==-1) {
                                            bestOS = r.os;
                                        }
                                        break;
                                    }
                                }
                                if (bestFamily!=null) {
                                    this.hostname = "(?)"+bestFamily;
                                } else if (bestOS!=null) {
                                    this.hostname = "(?)"+bestOS;
                                }
                                log.info(this.o.name,this.hostname);
                                if (this.hostname!=null && this.o.name!=this.hostname) {
                                    //this.o.name = this.hostname;
                                    this.save("name",null);
                                }
                            }
                        });
            */

        }
    }

    hasBeenGivenName() {
        if (this.o.name == null) {
            return false;
        }
        if (this.o.name == this.o.ipv4Addr || this.o.name.indexOf("(?)") != -1 || this.o.name == "undefined") {
            return false;
        }
        return true;
    }

    saveAsync(tuple) {
        return new Promise((resolve, reject) => {
            this.save(tuple,(err, data) => {
                if(err) {
                    reject(err);
                } else {
                    resolve(data);
                }
            });
        });
    }

    save(tuple, callback) {
        if (tuple == null) {
            this.redisfy();
            rclient.hmset("host:mac:" + this.o.mac, this.o, (err) => {
                if (callback) {
                    callback(err);
                }
            });
        } else {
            this.redisfy();
            log.debug("Saving ", this.o.ipv4Addr, tuple, this.o[tuple]);
            let obj = {};
            obj[tuple] = this.o[tuple];
            rclient.hmset("host:mac:" + this.o.mac, obj, (err) => {
                if (callback) {
                    callback(err);
                }
            });
        }
    }

    setAdmin(tuple, value) {
        if (this.admin == null) {
            this.admin = {};
        }
        this.admin[tuple] = value;
        this.redisfy();

        rclient.hmset("host:mac:" + this.o.mac, {
            'admin': this.o.admin
        });
    }

    getAdmin(tuple) {
        if (this.admin == null) {
            return null;
        }
        return this.admin[tuple];
    }

    parse() {
        if (this.o.ipv6Addr) {
            this.ipv6Addr = JSON.parse(this.o.ipv6Addr);
        } else {}
        if (this.o.admin) {
            this.admin = JSON.parse(this.o.admin);
        } else {}
        if (this.o.dtype) {
            this.dtype = JSON.parse(this.o.dtype);
        }
        if (this.o.activities) {
            this.activities= JSON.parse(this.o.activities);
        }
    }

    redisfy() {
        if (this.ipv6Addr) {
            this.o.ipv6Addr = JSON.stringify(this.ipv6Addr);
        }
        if (this.admin) {
            this.o.admin = JSON.stringify(this.admin);
        }
        if (this.dtype) {
            this.o.dtype = JSON.stringify(this.dtype);
        }
        if (this.activities) {
            this.o.activities= JSON.stringify(this.activities);
        }
    }

    touch(date) {
        if (date != null || date <= this.o.lastActiveTimestamp) {
            return;
        }
        if (date == null) {
            date = Date.now() / 1000;
        }
        this.o.lastActiveTimestamp = date;
        rclient.hmset("host:mac:" + this.o.mac, {
            'lastActiveTimestamp': this.o.lastActiveTimestamp
        });
    }

  getAllIPs() {
    let list = [];
    list.push(this.o.ipv4Addr);
    if (this.ipv6Addr && this.ipv6Addr.length > 0) {
      for (let j in this['ipv6Addr']) {
        list.push(this['ipv6Addr'][j]);
      }
    }
    return list;
  }


    spoof(state) {
        log.debug("Spoofing ", this.o.ipv4Addr, this.ipv6Addr, this.o.mac, state, this.spoofing);
        if (this.o.ipv4Addr == null) {
            log.info("Host:Spoof:NoIP", this.o);
            return;
        }
        log.info("Host:Spoof:", this.o.name, this.o.ipv4Addr, this.o.mac, state, this.spoofing);
        let gateway = sysManager.monitoringInterface().gateway;
        let gateway6 = sysManager.monitoringInterface().gateway6;

      if(fConfig.newSpoof) {
        // new spoof supports spoofing on same device for mutliple times,
        // so no need to check if it is already spoofing or not
        if (this.o.ipv4Addr === gateway || this.o.mac == null || this.o.ipv4Addr === sysManager.myIp()) {
          return;
        }
        if (this.o.mac == "00:00:00:00:00:00" || this.o.mac.indexOf("00:00:00:00:00:00")>-1) {
          return;
        }
        if (this.o.mac == "FF:FF:FF:FF:FF:FF" || this.o.mac.indexOf("FF:FF:FF:FF:FF:FF")>-1) {
          return;
        }
        if(state === true) {
          hostTool.getMacByIP(gateway).then((gatewayMac) => {
            if (gatewayMac && gatewayMac === this.o.mac) {
              // ignore devices that has same mac address as gateway
              log.info(this.o.ipv4Addr + " has same mac address as gateway. Skip spoofing...");
              return;
            }
            spoofer.newSpoof(this.o.ipv4Addr)
            .then(() => {
              rclient.hmsetAsync("host:mac:" + this.o.mac, 'spoofing', true, 'spoofingTime', new Date() / 1000)
                .catch(err => log.error("Unable to set spoofing in redis", err))
                .then(() => this.dnsmasq.onSpoofChanged());
              log.info("Started spoofing", this.o.ipv4Addr, this.o.mac, this.o.name);
              this.spoofing = true;
            }).catch((err) => {
              log.error("Failed to spoof", this.o.ipv4Addr, this.o.mac, this.o.name);
            })
          })
        } else {
          spoofer.newUnspoof(this.o.ipv4Addr)
            .then(() => {
              rclient.hmsetAsync("host:mac:" + this.o.mac, 'spoofing', false, 'unspoofingTime', new Date() / 1000)
                .catch(err => log.error("Unable to set spoofing in redis", err))
                .then(() => this.dnsmasq.onSpoofChanged());
              log.debug("Stopped spoofing", this.o.ipv4Addr, this.o.mac, this.o.name);
              this.spoofing = false;
            }).catch((err) => {
            log.error("Failed to unspoof", this.o.ipv4Addr, this.o.mac, this.o.name);
          })
        }

        /* put a safety on the spoof */
        log.debug("Spoof For IPv6",this.o.mac, JSON.stringify(this.ipv6Addr),JSON.stringify(this.o.ipv6Addr),{});
        let myIp6 = sysManager.myIp6();
        if (this.ipv6Addr && this.ipv6Addr.length>0) {
            for (let i in this.ipv6Addr) {
                if (this.ipv6Addr[i] == gateway6) {
                    continue;
                }
                if (myIp6 && myIp6.indexOf(this.ipv6Addr[i])>-1) {
                    continue;
                }
                if (state == true) {
                    spoofer.newSpoof6(this.ipv6Addr[i]).then(()=>{
                         log.debug("Starting v6 spoofing", this.ipv6Addr[i]);
                    }).catch((err)=>{
                         log.error("Failed to spoof", this.ipv6Addr);
                    })
                    if (i>20) {
                         log.error("Failed to Spoof, over ",i, " of ", this.ipv6Addr,{});
                         break;
                    }
                    // prototype
                 //   log.debug("Host:Spoof:True", this.o.ipv4Addr, gateway,this.ipv6Addr,gateway6);
                 //   spoofer.spoof(null, null, this.o.mac, this.ipv6Addr,gateway6);
                 //   this.spoofing = true;
                } else {
                    spoofer.newUnspoof6(this.ipv6Addr[i]).then(()=>{
                         log.debug("Starting v6 unspoofing", this.ipv6Addr[i]);
                    }).catch((err)=>{
                         log.error("Failed to [v6] unspoof", this.ipv6Addr);
                    })
                //    log.debug("Host:Spoof:False", this.o.ipv4Addr, gateway, this.ipv6Addr,gateway6);
                //    spoofer.unspoof(null, null, this.o.mac,this.ipv6Addr, gateway6);
                //    this.spoofing = false;
                }
           }
        }
      } else {
/*
        if (state === true && this.spoofing === false) {
          log.info("Host:Spoof:True", this.o.ipv4Addr, gateway,this.ipv6Addr,gateway6);
          spoofer.spoof(this.o.ipv4Addr, gateway, this.o.mac, this.ipv6Addr,gateway6);
          this.spoofing = true;
        } else if (state === false && this.spoofing === true) {
          log.info("Host:Spoof:False", this.o.ipv4Addr, gateway, this.ipv6Addr,gateway6);
          spoofer.unspoof(this.o.ipv4Addr, gateway, this.o.mac,this.ipv6Addr, gateway6);
          this.spoofing = false;
        }
*/
      }
    }

    // Notice
    processNotifications(channel, message) {
        log.debug("RX Notifcaitons", channel, message);
        if (channel.toLowerCase().indexOf("notice") >= 0) {
            if (this.callbacks.notice != null) {
                this.callbacks.notice(this, channel, message);
            }
        }
    }

    /*
    {"ts":1466353908.736661,"uid":"CYnvWc3enJjQC9w5y2","id.orig_h":"192.168.2.153","id.orig_p":58515,"id.resp_h":"172.16.17.32","id.resp_p":80,"seen.indicator":"streamhd24.com","seen
    .indicator_type":"Intel::DOMAIN","seen.where":"HTTP::IN_HOST_HEADER","seen.node":"bro","sources":["from http://spam404bl.com/spam404scamlist.txt via intel.criticalstack.com"]}
    */
    subscribe(ip, e) {
        this.subscriber.subscribe("DiscoveryEvent", e, ip, (channel, type, ip, obj) => {
            log.debug("Host:Subscriber", channel, type, ip, obj);
            if (type == "Notice:Detected") {
                if (this.callbacks[e]) {
                    this.callbacks[e](channel, ip, type, obj);
                }
            } else if (type == "Intel:Detected") {
                // no need to handle intel here.                
            } else if (type == "HostPolicy:Changed" && this.type == "server") {
                this.applyPolicy((err)=>{
                });
                log.info("HostPolicy:Changed", channel, ip, type, obj);
            }<commit_msg>bug fix<commit_after>/*    Copyright 2016 Firewalla LLC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';
let log = require('./logger.js')(__filename);

var iptool = require('ip');
var os = require('os');
var network = require('network');
var instances = {};

const rclient = require('../util/redis_manager.js').getRedisClient()
const sclient = require('../util/redis_manager.js').getSubscriptionClient()

const exec = require('child-process-promise').exec

let Promise = require('bluebird');

const timeSeries = require('../util/TimeSeries.js').getTimeSeries()
const getHitsAsync = Promise.promisify(timeSeries.getHits).bind(timeSeries)

var Spoofer = require('./Spoofer.js');
var spoofer = null;
var SysManager = require('./SysManager.js');
var sysManager = new SysManager('info');
var DNSManager = require('./DNSManager.js');
var dnsManager = new DNSManager('error');
var FlowManager = require('./FlowManager.js');
var flowManager = new FlowManager('debug');

const DNSMASQ = require('../extension/dnsmasq/dnsmasq.js');

const FRPManager = require('../extension/frp/FRPManager.js')
const fm = new FRPManager()
const frp = fm.getSupportFRP()

const AlarmManager2 = require('../alarm/AlarmManager2.js');
const alarmManager2 = new AlarmManager2();

const PolicyManager2 = require('../alarm/PolicyManager2.js');
const policyManager2 = new PolicyManager2();
const pm2 = policyManager2

let ExceptionManager = require('../alarm/ExceptionManager.js');
let exceptionManager = new ExceptionManager();

let spooferManager = require('./SpooferManager.js')

let modeManager = require('./ModeManager.js');

let async = require('asyncawait/async');
let await = require('asyncawait/await');

let f = require('./Firewalla.js');

const getPreferredBName = require('../util/util.js').getPreferredBName

const license = require('../util/license.js')

var alarmManager = null;

var uuid = require('uuid');
var bone = require("../lib/Bone.js");

var utils = require('../lib/utils.js');

let fConfig = require('./config.js').getConfig();

const fc = require('./config.js')

var _async = require('async');

var MobileDetect = require('mobile-detect');

var flowUtil = require('../net2/FlowUtil.js');

let AppTool = require('./AppTool');
let appTool = new AppTool();

var linux = require('../util/linux.js');

const HostTool = require('../net2/HostTool.js')
const hostTool = new HostTool()

/* alarms:
    alarmtype:  intel/newhost/scan/log
    severityscore: out of 100
    alarmseverity: major minor
*/


class Host {
    constructor(obj, mgr, callback) {
      this.callbacks = {};
      this.o = obj;
      this.mgr = mgr;
      if (this.o.ipv4) {
        this.o.ipv4Addr = this.o.ipv4;
      }

      this._mark = false;
      this.parse();

      let c = require('./MessageBus.js');
      this.subscriber = new c('debug');

      if (this.mgr.type === 'server') {
        this.spoofing = false;
        sclient.on("message", (channel, message) => {
          this.processNotifications(channel, message);
        });

        if (obj != null) {
          this.subscribe(this.o.ipv4Addr, "Notice:Detected");
          this.subscribe(this.o.ipv4Addr, "Intel:Detected");
          this.subscribe(this.o.ipv4Addr, "HostPolicy:Changed");
        }
        this.spoofing = false;

        /*
         if (this.o.ipv6Addr) {
         this.o.ipv6Addr = JSON.parse(this.o.ipv6Addr);
         }
         */
        this.predictHostNameUsingUserAgent();

        this.loadPolicy(callback);
      }

      this.dnsmasq = new DNSMASQ();
    }

    update(obj) {
        this.o = obj;
        if (this.o.ipv4) {
            this.o.ipv4Addr = this.o.ipv4;
        }

        if(this.mgr.type === 'server') {
          if (obj != null) {
            this.subscribe(this.o.ipv4Addr, "Notice:Detected");
            this.subscribe(this.o.ipv4Addr, "Intel:Detected");
            this.subscribe(this.o.ipv4Addr, "HostPolicy:Changed");
          }
          this.predictHostNameUsingUserAgent();
          this.loadPolicy(null);
        }

        this.parse();
    }

/* example of ipv6Host
1) "mac"
2) "B8:53:AC:5F:99:51"
3) "firstFoundTimestamp"
4) "1511599097.786"
5) "lastActiveTimestamp"
6) "1511846844.798"

*/

    keepalive() {
        for (let i in this.ipv6Addr) {
            log.debug("keep alive ", this.mac,this.ipv6Addr[i]);
            linux.ping6(null,this.ipv6Addr[i]);
        }
        setTimeout(()=>{
            this.cleanV6();
        },1000*10);
    }

    cleanV6() {
        return async(()=> {
            if (this.ipv6Addr == null) {
                return;
            }

            let ts = (new Date())/1000;
            let lastActive = 0;
            let _ipv6Hosts = {};
            this._ipv6Hosts = {};

            for (let i in this.ipv6Addr) {
                let ip6 = this.ipv6Addr[i];
                let ip6Host = await(rclient.hgetallAsync("host:ip6:"+ip6));
                log.debug("HostManager:CleanV6:looking up v6",ip6,ip6Host)
                if (ip6Host != null) {
                    _ipv6Hosts[ip6] = ip6Host;
                    if (ip6Host.lastActiveTimestamp > lastActive) {
                        lastActive = ip6Host.lastActiveTimestamp;
                    }
                }
            }

            this.ipv6Addr = [];
            for (let ip6 in _ipv6Hosts) {
                let ip6Host = _ipv6Hosts[ip6];
                if (ip6Host.lastActiveTimestamp < lastActive - 60*30 || ip6Host.lastActiveTimestamp < ts-60*40) {
                    log.info("Host:"+this.mac+","+ts+","+ip6Host.lastActiveTimestamp+","+lastActive+" Remove Old Address "+ip6,JSON.stringify(ip6Host));
                } else {
                    this._ipv6Hosts[ip6] = ip6Host;
                    this.ipv6Addr.push(ip6);
                }
            }

            if (this.o.lastActiveTimestamp < lastActive) {
                this.o.lastActiveTimestamp = lastActive;
            }

            //await(this.saveAsync());
            log.debug("HostManager:CleanV6:", this.o.mac, JSON.stringify(this.ipv6Addr));
        })().catch((err) => {
            log.error("Got error when cleanV6", err, {})            
        });
    }

    predictHostNameUsingUserAgent() {
        if (this.hasBeenGivenName() == false) {
            rclient.smembers("host:user_agent_m:" + this.o.mac, (err, results) => {
                if (results != null && results.length > 0) {
                    let familydb = {};
                    let osdb = {};
                    let mobile = false;
                    let md_osdb = {};
                    let md_name = {};

                    for (let i in results) {
                        let r = JSON.parse(results[i]);
                        if (r.ua) {
                            let md = new MobileDetect(r.ua);
                            if (md == null) {
                                log.info("MD Null");
                                continue;
                            }
                            let name = null;
                            if (md.mobile()) {
                                mobile = true;
                                name = md.mobile();
                            }
                            let os = md.os();
                            if (os != null) {
                                if (md_osdb[os]) {
                                    md_osdb[os] += 1;
                                } else {
                                    md_osdb[os] = 1;
                                }
                            }
                            if (name != null) {
                                if (md_name[name]) {
                                    md_name[name] += 1;
                                } else {
                                    md_name[name] = 1;
                                }
                            }
                        }

                        /*
                        if (r!=null) {
                            if (r.family.indexOf("Other")==-1) {
                                if (familydb[r.family]) {
                                   familydb[r.family] += 1;
                                } else {
                                   familydb[r.family] = 1;
                                }
                                bestFamily = r.family;
                            } else if (r.os.indexOf("Other")==-1) {
                                bestOS = r.os;
                            }
                            break;
                        }
                        */
                    }
                    log.debug("Sorting", JSON.stringify(md_name), JSON.stringify(md_osdb))
                    let bestOS = null;
                    let bestName = null;
                    let osarray = [];
                    let namearray = [];
                    for (let i in md_osdb) {
                        osarray.push({
                            name: i,
                            rank: md_osdb[i]
                        });
                    }
                    for (let i in md_name) {
                        namearray.push({
                            name: i,
                            rank: md_name[i]
                        });
                    }
                    osarray.sort(function (a, b) {
                        return Number(b.rank) - Number(a.rank);
                    })
                    namearray.sort(function (a, b) {
                        return Number(b.rank) - Number(a.rank);
                    })
                    if (namearray.length > 0) {
                        this.o.ua_name = namearray[0].name;
                        this.predictedName = "(?)" + this.o.ua_name;

                        if (osarray.length > 0) {
                            this.o.ua_os_name = osarray[0].name;
                            this.predictedName += "/" + this.o.ua_os_name;
                        }
                        this.o.pname = this.predictedName;
                        log.debug(">>>>>>>>>>>> ", this.predictedName, JSON.stringify(this.o.ua_os_name));
                    }
                    if (mobile == true) {
                        this.o.deviceClass = "mobile";
                        this.save("deviceClass", null);
                    }
                    if (this.o.ua_os_name) {
                        this.save("ua_os_name", null);
                    }
                    if (this.o.ua_name) {
                        this.save("ua_name", null);
                    }
                    if (this.o.pname) {
                        this.save("pname", null);
                    }
                    //this.save("ua_name",null);

                    /*
                    if (bestFamily!=null) {
                        this.hostname = "(?)"+bestFamily;
                    } else if (bestOS!=null) {
                        this.hostname = "(?)"+bestOS;
                    }
                    log.info(this.o.name,this.hostname);
                    if (this.hostname!=null && this.o.name!=this.hostname) {
                        //this.o.name = this.hostname;
                        this.save("name",null);
                    }
                   */
                }
            });

            /*
                        rclient.smembers("host:user_agent:"+this.o.ipv4Addr,(err,results)=> {
                            if (results!=null && results.length>0) {
                                let bestFamily = null;
                                let bestOS = null;
                                for (let i in results) {
                                    let r = JSON.parse(results[i]);
                                    if (r!=null) {
                                        if (r.family.indexOf("Other")==-1) {
                                            bestFamily = r.family;
                                        } else if (r.os.indexOf("Other")==-1) {
                                            bestOS = r.os;
                                        }
                                        break;
                                    }
                                }
                                if (bestFamily!=null) {
                                    this.hostname = "(?)"+bestFamily;
                                } else if (bestOS!=null) {
                                    this.hostname = "(?)"+bestOS;
                                }
                                log.info(this.o.name,this.hostname);
                                if (this.hostname!=null && this.o.name!=this.hostname) {
                                    //this.o.name = this.hostname;
                                    this.save("name",null);
                                }
                            }
                        });
            */

        }
    }

    hasBeenGivenName() {
        if (this.o.name == null) {
            return false;
        }
        if (this.o.name == this.o.ipv4Addr || this.o.name.indexOf("(?)") != -1 || this.o.name == "undefined") {
            return false;
        }
        return true;
    }

    saveAsync(tuple) {
        return new Promise((resolve, reject) => {
            this.save(tuple,(err, data) => {
                if(err) {
                    reject(err);
                } else {
                    resolve(data);
                }
            });
        });
    }

    save(tuple, callback) {
        if (tuple == null) {
            this.redisfy();
            rclient.hmset("host:mac:" + this.o.mac, this.o, (err) => {
                if (callback) {
                    callback(err);
                }
            });
        } else {
            this.redisfy();
            log.debug("Saving ", this.o.ipv4Addr, tuple, this.o[tuple]);
            let obj = {};
            obj[tuple] = this.o[tuple];
            rclient.hmset("host:mac:" + this.o.mac, obj, (err) => {
                if (callback) {
                    callback(err);
                }
            });
        }
    }

    setAdmin(tuple, value) {
        if (this.admin == null) {
            this.admin = {};
        }
        this.admin[tuple] = value;
        this.redisfy();

        rclient.hmset("host:mac:" + this.o.mac, {
            'admin': this.o.admin
        });
    }

    getAdmin(tuple) {
        if (this.admin == null) {
            return null;
        }
        return this.admin[tuple];
    }

    parse() {
        if (this.o.ipv6Addr) {
            this.ipv6Addr = JSON.parse(this.o.ipv6Addr);
        } else {}
        if (this.o.admin) {
            this.admin = JSON.parse(this.o.admin);
        } else {}
        if (this.o.dtype) {
            this.dtype = JSON.parse(this.o.dtype);
        }
        if (this.o.activities) {
            this.activities= JSON.parse(this.o.activities);
        }
    }

    redisfy() {
        if (this.ipv6Addr) {
            this.o.ipv6Addr = JSON.stringify(this.ipv6Addr);
        }
        if (this.admin) {
            this.o.admin = JSON.stringify(this.admin);
        }
        if (this.dtype) {
            this.o.dtype = JSON.stringify(this.dtype);
        }
        if (this.activities) {
            this.o.activities= JSON.stringify(this.activities);
        }
    }

    touch(date) {
        if (date != null || date <= this.o.lastActiveTimestamp) {
            return;
        }
        if (date == null) {
            date = Date.now() / 1000;
        }
        this.o.lastActiveTimestamp = date;
        rclient.hmset("host:mac:" + this.o.mac, {
            'lastActiveTimestamp': this.o.lastActiveTimestamp
        });
    }

  getAllIPs() {
    let list = [];
    list.push(this.o.ipv4Addr);
    if (this.ipv6Addr && this.ipv6Addr.length > 0) {
      for (let j in this['ipv6Addr']) {
        list.push(this['ipv6Addr'][j]);
      }
    }
    return list;
  }


    spoof(state) {
        log.debug("Spoofing ", this.o.ipv4Addr, this.ipv6Addr, this.o.mac, state, this.spoofing);
        if (this.o.ipv4Addr == null) {
            log.info("Host:Spoof:NoIP", this.o);
            return;
        }
        log.info("Host:Spoof:", this.o.name, this.o.ipv4Addr, this.o.mac, state, this.spoofing);
        let gateway = sysManager.monitoringInterface().gateway;
        let gateway6 = sysManager.monitoringInterface().gateway6;

      if(fConfig.newSpoof) {
        // new spoof supports spoofing on same device for mutliple times,
        // so no need to check if it is already spoofing or not
        if (this.o.ipv4Addr === gateway || this.o.mac == null || this.o.ipv4Addr === sysManager.myIp()) {
          return;
        }
        if (this.o.mac == "00:00:00:00:00:00" || this.o.mac.indexOf("00:00:00:00:00:00")>-1) {
          return;
        }
        if (this.o.mac == "FF:FF:FF:FF:FF:FF" || this.o.mac.indexOf("FF:FF:FF:FF:FF:FF")>-1) {
          return;
        }
        if(state === true) {
          hostTool.getMacByIP(gateway).then((gatewayMac) => {
            if (gatewayMac && gatewayMac === this.o.mac) {
              // ignore devices that has same mac address as gateway
              log.info(this.o.ipv4Addr + " has same mac address as gateway. Skip spoofing...");
              return;
            }
            spoofer.newSpoof(this.o.ipv4Addr)
            .then(() => {
              rclient.hmsetAsync("host:mac:" + this.o.mac, 'spoofing', true, 'spoofingTime', new Date() / 1000)
                .catch(err => log.error("Unable to set spoofing in redis", err))
                .then(() => this.dnsmasq.onSpoofChanged());
              log.info("Started spoofing", this.o.ipv4Addr, this.o.mac, this.o.name);
              this.spoofing = true;
            }).catch((err) => {
              log.error("Failed to spoof", this.o.ipv4Addr, this.o.mac, this.o.name);
            })
          })
        } else {
          spoofer.newUnspoof(this.o.ipv4Addr)
            .then(() => {
              rclient.hmsetAsync("host:mac:" + this.o.mac, 'spoofing', false, 'unspoofingTime', new Date() / 1000)
                .catch(err => log.error("Unable to set spoofing in redis", err))
                .then(() => this.dnsmasq.onSpoofChanged());
              log.debug("Stopped spoofing", this.o.ipv4Addr, this.o.mac, this.o.name);
              this.spoofing = false;
            }).catch((err) => {
            log.error("Failed to unspoof", this.o.ipv4Addr, this.o.mac, this.o.name);
          })
        }

        /* put a safety on the spoof */
        log.debug("Spoof For IPv6",this.o.mac, JSON.stringify(this.ipv6Addr),JSON.stringify(this.o.ipv6Addr),{});
        let myIp6 = sysManager.myIp6();
        if (this.ipv6Addr && this.ipv6Addr.length>0) {
            for (let i in this.ipv6Addr) {
                if (this.ipv6Addr[i] == gateway6) {
                    continue;
                }
                if (myIp6 && myIp6.indexOf(this.ipv6Addr[i])>-1) {
                    continue;
                }
                if (state == true) {
                    spoofer.newSpoof6(this.ipv6Addr[i]).then(()=>{
                         log.debug("Starting v6 spoofing", this.ipv6Addr[i]);
                    }).catch((err)=>{
                         log.error("Failed to spoof", this.ipv6Addr);
                    })
                    if (i>20) {
                         log.error("Failed to Spoof, over ",i, " of ", this.ipv6Addr,{});
                         break;
                    }
                    // prototype
                 //   log.debug("Host:Spoof:True", this.o.ipv4Addr, gateway,this.ipv6Addr,gateway6);
                 //   spoofer.spoof(null, null, this.o.mac, this.ipv6Addr,gateway6);
                 //   this.spoofing = true;
                } else {
                    spoofer.newUnspoof6(this.ipv6Addr[i]).then(()=>{
                         log.debug("Starting v6 unspoofing", this.ipv6Addr[i]);
                    }).catch((err)=>{
                         log.error("Failed to [v6] unspoof", this.ipv6Addr);
                    })
                //    log.debug("Host:Spoof:False", this.o.ipv4Addr, gateway, this.ipv6Addr,gateway6);
                //    spoofer.unspoof(null, null, this.o.mac,this.ipv6Addr, gateway6);
                //    this.spoofing = false;
                }
           }
        }
      } else {
/*
        if (state === true && this.spoofing === false) {
          log.info("Host:Spoof:True", this.o.ipv4Addr, gateway,this.ipv6Addr,gateway6);
          spoofer.spoof(this.o.ipv4Addr, gateway, this.o.mac, this.ipv6Addr,gateway6);
          this.spoofing = true;
        } else if (state === false && this.spoofing === true) {
          log.info("Host:Spoof:False", this.o.ipv4Addr, gateway, this.ipv6Addr,gateway6);
          spoofer.unspoof(this.o.ipv4Addr, gateway, this.o.mac,this.ipv6Addr, gateway6);
          this.spoofing = false;
        }
*/
      }
    }

    // Notice
    processNotifications(channel, message) {
        log.debug("RX Notifcaitons", channel, message);
        if (channel.toLowerCase().indexOf("notice") >= 0) {
            if (this.callbacks.notice != null) {
                this.callbacks.notice(this, channel, message);
            }
        }
    }

    /*
    {"ts":1466353908.736661,"uid":"CYnvWc3enJjQC9w5y2","id.orig_h":"192.168.2.153","id.orig_p":58515,"id.resp_h":"172.16.17.32","id.resp_p":80,"seen.indicator":"streamhd24.com","seen
    .indicator_type":"Intel::DOMAIN","seen.where":"HTTP::IN_HOST_HEADER","seen.node":"bro","sources":["from http://spam404bl.com/spam404scamlist.txt via intel.criticalstack.com"]}
    */
    subscribe(ip, e) {
        this.subscriber.subscribe("DiscoveryEvent", e, ip, (channel, type, ip2, obj) => {
            log.debug("Host:Subscriber", channel, type, ip2, obj);
            if (type === "Notice:Detected") {
                if (this.callbacks[e]) {
                    this.callbacks[e](channel, ip2, type, obj);
                }
            } else if (type === "Intel:Detected") {
                // no need to handle intel here.                
            } else if (type === "HostPolicy:Changed" && this.type === "server") {
                this.applyPolicy((err)=>{
                });
                log.info("HostPolicy:Changed", channel, ip, ip2, type, obj);
            }
{'fraction_non_alphanumeric': 0.1373291871058164, 'fraction_numerical': 0.014540995094604065, 'mean_word_length': 1.29023521741311, 'pattern_counts': {'":': 22, '<': 13, '<?xml version=': 0, '>': 103, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 76, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4898755', 'n_tokens_mistral': 13708, 'n_tokens_neox': 13156, 'n_words': 2370}