<commit_before>/*    Copyright 2019-2021 Firewalla Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

'use strict';

// this module should be responsible for connecting firewalla and fireroute together
// more specifically:
//    inquery and update regarding to
//      physical/bridge/virtual interfaces, DNS, NAT, DHCP
//    serving as FireRouter on RED & BLUE
//      create secondaryInterface
//      start bro, dnsmasq if necessary
//      generating compatible config


//  An Interface class?
//  {
//    name
//    gateway: {ipv4, ipv4}
//    subnet
//    dnsmasq as a pre-interface instance?
//  }

const log = require("./logger.js")(__filename);

const f = require('../net2/Firewalla.js');
const SysTool = require('../net2/SysTool.js')
const sysTool = new SysTool()
const broControl = require('./BroControl.js')
const PlatformLoader = require('../platform/PlatformLoader.js')
const platform = PlatformLoader.getPlatform()
const Config = require('./config.js')
const rclient = require('../util/redis_manager.js').getRedisClient()
const { delay } = require('../util/util.js')
const pclient = require('../util/redis_manager.js').getPublishClient();
const sclient = require('../util/redis_manager.js').getSubscriptionClient();
const Message = require('./Message.js');
const Mode = require('./Mode.js');
const sem = require('../sensor/SensorEventManager.js').getInstance();

const util = require('util')
const rp = util.promisify(require('request'))
const { Address4, Address6 } = require('ip-address')
const ip = require('ip')
const _ = require('lodash');
const exec = require('child-process-promise').exec;
const era = require('../event/EventRequestApi.js');
const AsyncLock = require('../vendor_lib/async-lock');
const lock = new AsyncLock();
const LOCK_INIT = "LOCK_INIT";

// not exposing these methods/properties
async function localGet(endpoint) {
  if (!platform.isFireRouterManaged())
    throw new Error('Forbidden')

  let retry = 10;
  while (retry > 0) {
    try {
      const options = {
        method: "GET",
        headers: {
          "Accept": "application/json"
        },
        url: routerInterface + endpoint,
        json: true
      };

      const resp = await rp(options)
      if (resp.statusCode !== 200) {
        throw new Error(`Error getting ${endpoint}, code: ${resp.statusCode}`);
      }

      return resp.body
    } catch (err) {
      retry -= 1;
      log.error(`Failed to get ${endpoint} from firerouter API, ${retry > 0 ? "will try again later" : "skip due to too many failed retries"}`, err.message);
      await delay(2000);
    }
  }
}

async function getConfig() {
  return localGet("/config/active")
}

async function getWANInterfaces() {
  return localGet("/config/wans")
}

async function getLANInterfaces() {
  return localGet("/config/lans")
}

async function getInterfaces() {
  return localGet("/config/interfaces")
}

function updateMaps() {
  for (const intfName in intfNameMap) {
    const intf = intfNameMap[intfName]
    // this usually happens after consecutive network config update, internal data structure of interface in firerouter is incomplete
    if (!intf.config || !intf.config.meta) {
      log.error(`Interface ${intfName} does not have config or config.meta`)
      return false;
    }
    intf.config.meta.intfName = intfName
    intfUuidMap[intf.config.meta.uuid] = intf
  }
  return true;
}

function calculateLocalNetworks(monitoringInterfaces, sysNetworkInfo) {
  const localNetworks = {};
  // add multicast ip range to local networks so that related traffic will be marked as local_resp/local_orig:true and will be directly bypassed in BroDetect.js
  const multicastV4 = "172.16.31.10/4";
  const multicastV6 = "ff00::/8";
  for (const intf of sysNetworkInfo) {
    const intfName = intf.name;
    if (!monitoringInterfaces.includes(intfName))
      continue;
    if (intf.ip4_subnets && _.isArray(intf.ip4_subnets)) {
      for (const ip of intf.ip4_subnets) {
        if (localNetworks[ip])
          localNetworks[ip].push(intfName);
        else
          localNetworks[ip] = [intfName];
      }
    }
    if (localNetworks[multicastV4])
      localNetworks[multicastV4].push(intfName);
    else
      localNetworks[multicastV4] = [intfName];
    if (intf.ip6_subnets && _.isArray(intf.ip6_subnets)) {
      for (const ip of intf.ip6_subnets) {
        if (localNetworks[ip])
          localNetworks[ip].push(intfName);
        else
          localNetworks[ip] = [intfName];
      }
    }
    if (localNetworks[multicastV6])
      localNetworks[multicastV6].push(intfName);
    else
      localNetworks[multicastV6] = [intfName];
  }
  return localNetworks;
}

function getPcapBufsize(intfName) {
  const intfMatch = intfName.match(/^[^\d]+/)
  return intfMatch ? platform.getZeekPcapBufsize()[intfMatch[0]] : undefined
}

async function calculateZeekOptions(monitoringInterfaces) {
  const parentIntfOptions = {};
  const monitoringIntfOptions = {}
  for (const intfName in intfNameMap) {
    if (!monitoringInterfaces.includes(intfName))
      continue;
    const intf = intfNameMap[intfName];
    const subIntfs = intf.config && intf.config.intf;
    if (!subIntfs) {
      monitoringIntfOptions[intfName] = parentIntfOptions[intfName] = { pcapBufsize: getPcapBufsize(intfName) };<commit_msg>do not count bond's sub interfaces while calculating minimal listen ports of zeek in FireRouter.js<commit_after>/*    Copyright 2019-2021 Firewalla Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

'use strict';

// this module should be responsible for connecting firewalla and fireroute together
// more specifically:
//    inquery and update regarding to
//      physical/bridge/virtual interfaces, DNS, NAT, DHCP
//    serving as FireRouter on RED & BLUE
//      create secondaryInterface
//      start bro, dnsmasq if necessary
//      generating compatible config


//  An Interface class?
//  {
//    name
//    gateway: {ipv4, ipv4}
//    subnet
//    dnsmasq as a pre-interface instance?
//  }

const log = require("./logger.js")(__filename);

const f = require('../net2/Firewalla.js');
const SysTool = require('../net2/SysTool.js')
const sysTool = new SysTool()
const broControl = require('./BroControl.js')
const PlatformLoader = require('../platform/PlatformLoader.js')
const platform = PlatformLoader.getPlatform()
const Config = require('./config.js')
const rclient = require('../util/redis_manager.js').getRedisClient()
const { delay } = require('../util/util.js')
const pclient = require('../util/redis_manager.js').getPublishClient();
const sclient = require('../util/redis_manager.js').getSubscriptionClient();
const Message = require('./Message.js');
const Mode = require('./Mode.js');
const sem = require('../sensor/SensorEventManager.js').getInstance();

const util = require('util')
const rp = util.promisify(require('request'))
const { Address4, Address6 } = require('ip-address')
const ip = require('ip')
const _ = require('lodash');
const exec = require('child-process-promise').exec;
const era = require('../event/EventRequestApi.js');
const AsyncLock = require('../vendor_lib/async-lock');
const lock = new AsyncLock();
const LOCK_INIT = "LOCK_INIT";

// not exposing these methods/properties
async function localGet(endpoint) {
  if (!platform.isFireRouterManaged())
    throw new Error('Forbidden')

  let retry = 10;
  while (retry > 0) {
    try {
      const options = {
        method: "GET",
        headers: {
          "Accept": "application/json"
        },
        url: routerInterface + endpoint,
        json: true
      };

      const resp = await rp(options)
      if (resp.statusCode !== 200) {
        throw new Error(`Error getting ${endpoint}, code: ${resp.statusCode}`);
      }

      return resp.body
    } catch (err) {
      retry -= 1;
      log.error(`Failed to get ${endpoint} from firerouter API, ${retry > 0 ? "will try again later" : "skip due to too many failed retries"}`, err.message);
      await delay(2000);
    }
  }
}

async function getConfig() {
  return localGet("/config/active")
}

async function getWANInterfaces() {
  return localGet("/config/wans")
}

async function getLANInterfaces() {
  return localGet("/config/lans")
}

async function getInterfaces() {
  return localGet("/config/interfaces")
}

function updateMaps() {
  for (const intfName in intfNameMap) {
    const intf = intfNameMap[intfName]
    // this usually happens after consecutive network config update, internal data structure of interface in firerouter is incomplete
    if (!intf.config || !intf.config.meta) {
      log.error(`Interface ${intfName} does not have config or config.meta`)
      return false;
    }
    intf.config.meta.intfName = intfName
    intfUuidMap[intf.config.meta.uuid] = intf
  }
  return true;
}

function calculateLocalNetworks(monitoringInterfaces, sysNetworkInfo) {
  const localNetworks = {};
  // add multicast ip range to local networks so that related traffic will be marked as local_resp/local_orig:true and will be directly bypassed in BroDetect.js
  const multicastV4 = "172.16.31.10/4";
  const multicastV6 = "ff00::/8";
  for (const intf of sysNetworkInfo) {
    const intfName = intf.name;
    if (!monitoringInterfaces.includes(intfName))
      continue;
    if (intf.ip4_subnets && _.isArray(intf.ip4_subnets)) {
      for (const ip of intf.ip4_subnets) {
        if (localNetworks[ip])
          localNetworks[ip].push(intfName);
        else
          localNetworks[ip] = [intfName];
      }
    }
    if (localNetworks[multicastV4])
      localNetworks[multicastV4].push(intfName);
    else
      localNetworks[multicastV4] = [intfName];
    if (intf.ip6_subnets && _.isArray(intf.ip6_subnets)) {
      for (const ip of intf.ip6_subnets) {
        if (localNetworks[ip])
          localNetworks[ip].push(intfName);
        else
          localNetworks[ip] = [intfName];
      }
    }
    if (localNetworks[multicastV6])
      localNetworks[multicastV6].push(intfName);
    else
      localNetworks[multicastV6] = [intfName];
  }
  return localNetworks;
}

function getPcapBufsize(intfName) {
  const intfMatch = intfName.match(/^[^\d]+/)
  return intfMatch ? platform.getZeekPcapBufsize()[intfMatch[0]] : undefined
}

async function calculateZeekOptions(monitoringInterfaces) {
  const parentIntfOptions = {};
  const monitoringIntfOptions = {}
  for (const intfName in intfNameMap) {
    if (!monitoringInterfaces.includes(intfName))
      continue;
    const intf = intfNameMap[intfName];
    const isBond = intfName && intfName.startsWith("bond") && !intfName.includes(".");
    const subIntfs = !isBond && intf.config && intf.config.intf;
    if (!subIntfs) {
      monitoringIntfOptions[intfName] = parentIntfOptions[intfName] = { pcapBufsize: getPcapBufsize(intfName) };
{'fraction_non_alphanumeric': 0.12220453974369713, 'fraction_numerical': 0.009213501968339057, 'mean_word_length': 3.4140480591497226, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4288992', 'n_tokens_mistral': 3740, 'n_tokens_neox': 3593, 'n_words': 1002}