<commit_before>/*    Copyright 2020 Firewalla INC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('../net2/logger.js')(__filename);

const util = require('util');

const Sensor = require('./Sensor.js').Sensor;

const rclient = require('../util/redis_manager.js').getRedisClient();

const cp = require('child_process');

const Firewalla = require('../net2/Firewalla');

const xml2jsonBinary = Firewalla.getFirewallaHome() + "/extension/xml2json/xml2json." + Firewalla.getPlatform();

const fc = require('../net2/config.js');

const HostManager = require("../net2/HostManager.js");
const hostManager = new HostManager();

const sysManager = require('../net2/SysManager.js');
const sem = require('../sensor/SensorEventManager.js').getInstance();

class DeviceServiceScanSensor extends Sensor {
  constructor() {
    super();
  }

  run() {
    let firstScanTime = this.config.firstScan * 1000 || 120 * 1000; // default to 120 seconds
    setTimeout(async() => {
      await this.checkAndRunOnce();
      sem.emitEvent({
        type: "DeviceServiceScanComplete",
        message: ""
      });
    }, firstScanTime);

    let interval = this.config.interval * 1000 || 30 * 60 * 1000; // 30 minutes
    setInterval(() => {
      this.checkAndRunOnce();
    }, interval);
  }

  isSensorEnable() {
    return fc.isFeatureOn("device_service_scan");
  }

  async checkAndRunOnce() {
    try {
      let result = await this.isSensorEnable();
      if (result) {
        return await this.runOnce();
      };
    } catch(err) {
      log.error('Failed to scan: ', err);
    };

    return null;
  }

  async runOnce() {
    log.info('Scan start...');

    let results = await hostManager.getHostsAsync();
    if (!results)
      throw new Error('Failed to scan.');

    let hosts = [];
    try {
      results = results.filter((host) => host && host.o && host.o.mac && host.o.ipv4Addr && !sysManager.isMyIP(host.o.ipv4Addr));
      for (const host of results) {
        log.info("Scanning device: ", host.o.ipv4Addr);
        const scanResult = await this._scan(host.o.ipv4Addr);
        if (scanResult) {
          await rclient.hsetAsync("host:mac:" + host.o.mac, "openports", JSON.stringify(scanResult));
        }
      };
    } catch(err) {
      log.error("Failed to scan: " + err);
    }
    log.info('Scan finished...');
    return hosts;
  }

  _scan(ipAddr, callback) {
    let cmd = util.format('sudo nmap -Pn --top-ports 3000 %s -oX - | %s', ipAddr, xml2jsonBinary);

    log.info("Running command:", cmd);
    return new Promise((resolve, reject) => {
      cp.exec(cmd, (err, stdout, stderr) => {
        if(err || stderr) {
          reject(err || new Error(stderr));
          return;
        }

        let findings = null;
        try {
          findings = JSON.parse(stdout);
        } catch (err) {
          reject(err);
        }

        let nmapJSON = findings && findings.nmaprun && findings.nmaprun.host;
        resolve(this._parseNmapPortResult(nmapJSON));
      })
    });
  }

  static _handlePortEntry(portJson, openports) {
    if (portJson) {
      if (!openports[portJson.protocol])
        openports[portJson.protocol] = [];
      openports[portJson.protocol].push(portJson.portid * 1);
    }
  }

  _parseNmapPortResult(nmapResult) {
    let openports = {};
    openports.lastActiveTimestamp = Date.now() / 1000;
    try {
      let port = nmapResult && nmapResult.ports && nmapResult.ports.port;
      if (port && port.constructor === Object) {
        // one port only
        DeviceServiceScanSensor._handlePortEntry(port, openports);
      } else if (port && port.constructor === Array) {
        // multiple ports
        port.forEach((p) => DeviceServiceScanSensor._handlePortEntry(p, openports));
      }
    } catch(err) {
      log.error("Failed to parse nmap host: " + err);
    }
    return openports;
  }
}

module.exports = DeviceServiceScanSensor;<commit_msg>support per device/group port scan<commit_after>/*    Copyright 2020 Firewalla INC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('../net2/logger.js')(__filename);

const util = require('util');

const Sensor = require('./Sensor.js').Sensor;

const rclient = require('../util/redis_manager.js').getRedisClient();

const cp = require('child_process');

const Firewalla = require('../net2/Firewalla');

const xml2jsonBinary = Firewalla.getFirewallaHome() + "/extension/xml2json/xml2json." + Firewalla.getPlatform();

const fc = require('../net2/config.js');

const HostManager = require("../net2/HostManager.js");
const hostManager = new HostManager();

const sysManager = require('../net2/SysManager.js');
const sem = require('../sensor/SensorEventManager.js').getInstance();
const extensionManager = require('./ExtensionManager.js')
const featureName = "device_service_scan";
const policyKeyName = "device_service_scan";
class DeviceServiceScanSensor extends Sensor {
  constructor() {
    super();
  }

  async run() {
    this.scanSettings = {};
    const defaultOn = (await rclient.hgetAsync('policy:system', policyKeyName)) === null;
    this.scanSettings = {
      '0.0.0.0': defaultOn
    }
    let firstScanTime = this.config.firstScan * 1000 || 120 * 1000; // default to 120 seconds
    setTimeout(async () => {
      await this.checkAndRunOnce();
      sem.emitEvent({
        type: "DeviceServiceScanComplete",
        message: ""
      });
    }, firstScanTime);

    let interval = this.config.interval * 1000 || 30 * 60 * 1000; // 30 minutes
    setInterval(() => {
      this.checkAndRunOnce();
    }, interval);

    extensionManager.registerExtension(policyKeyName, this, {
      applyPolicy: this.applyPolicy
    });
  }

  isSensorEnable() {
    return fc.isFeatureOn(featureName);
  }
  async applyPolicy(host, ip, policy) {
    log.info("Applying device service scan policy:", ip, policy);
    try {
      if (ip === '0.0.0.0') {
        this.scanSettings[ip] = policy;
      } else {
        if (!host)
          return;
        switch (host.constructor.name) {
          case "Tag": {
            const tagUid = host.o && host.o.uid;
            if (tagUid) {
              const hosts = await hostManager.getHostsAsync();
              const tagHosts = hosts.filter((h) => {
                return h && h.o && h.o.tags && (
                  h.o.tags.includes(Number(tagUid)) || h.o.tags.includes(String(tagUid))
                ) && h.o.mac
              })
              tagHosts.map((h) => {
                !this.scanSettings[h.o.mac] && (this.scanSettings[h.o.mac] = {})
                this.scanSettings[h.o.mac].tagPolicy = policy;
              })
            }
            break;
          }
          case "NetworkProfile": {
            const uuid = host.o && host.o.uuid;
            if (uuid) {
              const hosts = await hostManager.getHostsAsync();
              const networkHosts = hosts.filter((h) => {
                return h && h.o && h.o.intf_uuid == uuid && h.o.mac
              })
              networkHosts.map((h) => {
                !this.scanSettings[h.o.mac] && (this.scanSettings[h.o.mac] = {})
                this.scanSettings[h.o.mac].networkPolicy = policy;
              })
            }
            break;
          }
          case "Host": {
            const macAddress = host && host.o && host.o.mac;
            if (!macAddress) return;
            !this.scanSettings[macAddress] && (this.scanSettings[macAddress] = {})
            this.scanSettings[macAddress].devicePolicy = policy;
            break;
          }
          default:
        }
      }
    } catch (err) {
      log.error("Got error when applying adblock policy", err);
    }
  }

  async checkAndRunOnce() {
    try {
      let result = await this.isSensorEnable();
      if (result) {
        return await this.runOnce();
      }
    } catch (err) {
      log.error('Failed to scan: ', err);
    }

    return null;
  }

  async runOnce() {
    log.info('Scan start...');

    let hosts = await hostManager.getHostsAsync();
    if (!hosts)
      throw new Error('Failed to scan.');

    try {
      hosts = hosts.filter((host) => {
        const validHost = host && host.o && host.o.mac && host.o.ipv4Addr && !sysManager.isMyIP(host.o.ipv4Addr);
        if (!validHost) return false;
        const mac = host.o.mac;
        const setting = this.scanSettings[mac];
        /* 
          same as adblock/familyProtect
          policy === null // exclude
          policy === false | undefined // unset
        */
        if (!setting) return this.scanSettings['0.0.0.0'];
        if (setting.devicePolicy === true) return true;
        if (setting.devicePolicy !== null && setting.tagPolicy === true) return true;
        if (setting.devicePolicy !== null && setting.networkPolicy === true) return true;
        if (setting.devicePolicy !== null && setting.tagPolicy !== null &&
          setting.networkPolicy !== null && this.scanSettings['0.0.0.0']) return true;
        return false;
      });
      for (const host of hosts) {
        log.info("Scanning device: ", host.o.ipv4Addr);
        const scanResult = await this._scan(host.o.ipv4Addr);
        if (scanResult) {
          await rclient.hsetAsync("host:mac:" + host.o.mac, "openports", JSON.stringify(scanResult));
        }
      }
    } catch (err) {
      log.error("Failed to scan: " + err);
    }
    log.info('Scan finished...');
    return hosts;
  }

  _scan(ipAddr) {
    let cmd = util.format('sudo nmap -Pn --top-ports 3000 %s -oX - | %s', ipAddr, xml2jsonBinary);

    log.info("Running command:", cmd);
    return new Promise((resolve, reject) => {
      cp.exec(cmd, (err, stdout, stderr) => {
        if (err || stderr) {
          reject(err || new Error(stderr));
          return;
        }

        let findings = null;
        try {
          findings = JSON.parse(stdout);
        } catch (err) {
          reject(err);
        }

        let nmapJSON = findings && findings.nmaprun && findings.nmaprun.host;
        resolve(this._parseNmapPortResult(nmapJSON));
      })
    });
  }

  static _handlePortEntry(portJson, openports) {
    if (portJson) {
      if (!openports[portJson.protocol])
        openports[portJson.protocol] = [];
      openports[portJson.protocol].push(portJson.portid * 1);
    }
  }

  _parseNmapPortResult(nmapResult) {
    let openports = {};
    openports.lastActiveTimestamp = Date.now() / 1000;
    try {
      let port = nmapResult && nmapResult.ports && nmapResult.ports.port;
      if (port && port.constructor === Object) {
        // one port only
        DeviceServiceScanSensor._handlePortEntry(port, openports);
      } else if (port && port.constructor === Array) {
        // multiple ports
        port.forEach((p) => DeviceServiceScanSensor._handlePortEntry(p, openports));
      }
    } catch (err) {
      log.error("Failed to parse nmap host: " + err);
    }
    return openports;
  }
}

module.exports = DeviceServiceScanSensor;
{'fraction_non_alphanumeric': 0.13848432485805973, 'fraction_numerical': 0.010367810417180942, 'mean_word_length': 2.3987695749440716, 'pattern_counts': {'":': 3, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 8}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2343375', 'n_tokens_mistral': 3696, 'n_tokens_neox': 3614, 'n_words': 882}