<commit_before>/*    Copyright 2016-2020 Firewalla Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */


/*
 * 2.2.15 RemoteHost
 *
 * This variable represents the source of inbound IP packets. This will be a wildcard in most cases
 * (i.e. an empty string). NAT vendors are only required to support wildcards. A non-wildcard value
 * will allow for “narrow” port mappings, which may be desirable in some usage scenarios.When
 * RemoteHost is a wildcard, all traffic sent to the ExternalPort on the WAN interface of the
 * gateway is forwarded to the InternalClient on the InternalPort. When RemoteHost is
 * specified as one external IP address as opposed to a wildcard, the NAT will only forward inbound
 * packets from this RemoteHost to the InternalClient, all other packets will be dropped.
 *
 * http://upnp.org/specs/gw/UPnP-gw-WANIPConnection-v1-Service.pdf
 */

'use strict';
const log = require('../net2/logger.js')(__filename);

const Sensor = require('./Sensor.js').Sensor;
const sem = require('./SensorEventManager').getInstance();

const rclient = require('../util/redis_manager.js').getRedisClient()

const UPNP = require('../extension/upnp/upnp.js');
const upnp = new UPNP();

const cfg = require('../net2/config.js');

const sysManager = require('../net2/SysManager.js');
const Message = require('../net2/Message.js');
const Alarm = require('../alarm/Alarm.js');
const AM2 = require('../alarm/AlarmManager2.js');
const am2 = new AM2();
const platform = require('../platform/PlatformLoader.js').getPlatform();
const fs = require('fs');
const exec = require('child-process-promise').exec;

const _ = require('lodash');

const ALARM_UPNP = 'alarm_upnp';

function compareUpnp(a, b) {
  return a.public && b.public &&
    a.private && b.private &&
    a.public.port == b.public.port &&
    a.private.host == b.private.host &&
    a.private.port == b.private.port &&
    a.protocol == b.protocol;
}

class UPNPSensor extends Sensor {
  constructor() {
    super();
    this.upnpLeaseFileWatchers = [];
  }

  isExpired(mapping) {
    const retentionPeriod = this.config.expireInterval || 1800;
    return !mapping.lastSeen || mapping.lastSeen < (Math.floor(Date.now() / 1000) - retentionPeriod) || (mapping.expire && mapping.expire < Math.floor(Date.now() / 1000));
  }

  mergeResults(curMappings, preMappings) {

    curMappings.forEach((mapping) => {
      mapping.expire = (mapping.ttl && mapping.ttl > 0) ? Math.floor(Date.now() / 1000) + mapping.ttl : null;
      mapping.lastSeen = Math.floor(Date.now() / 1000);
    });

    const fullMappings = [...curMappings, ...preMappings];

    const uniqMappings = _.uniqWith(fullMappings, compareUpnp);

    return uniqMappings
      .filter((mapping) => !this.isExpired(mapping));
  }

  scheduleReload() {
    if (this.reloadTask)
      clearTimeout(this.reloadTask);
    this.reloadTask = setTimeout(async () => {
      // UPnP lease file only exists on FireRouter enabled devices
      if (!platform.isFireRouterManaged())
        return;
      for (const watcher of this.upnpLeaseFileWatchers) {
        watcher.close();
      }
      this.upnpLeaseFileWatchers = [];
      const monitoringInterfaces = sysManager.getMonitoringInterfaces();
      for (const iface of monitoringInterfaces) {
        if (iface.name && iface.name.endsWith(":0"))
          continue;
        const leaseFile = `/var/run/upnp.${iface.name}.leases`;
        await exec(`sudo touch ${leaseFile}`).then(() => {
          const watcher = fs.watch(leaseFile, {}, (e, filename) => {
            if (e === "change") {
              log.info(`UPnP lease file ${leaseFile} is changed, schedule checking UPnP leases ...`);
              this.scheduleCheckUPnPLeases();
            }
            if (e === "rename") {
              log.info(`UPnP lease file ${leaseFile} is renamed, schedule reload UPNPSensor ...`);
              this.scheduleReload();
            }
          });
          log.info(`Watching UPnP lease file change on ${leaseFile} ...`);
          this.upnpLeaseFileWatchers.push(watcher);
        }).catch((err) => {
          log.error(`Failed to watch file change ${leaseFile}`, err.message);
        });
      }
    }, 5000);
  }

  scheduleCheckUPnPLeases() {
    if (this.checkTask)
      clearTimeout(this.checkTask);<commit_msg>immediately check UPNP leases in UPNPSensor.scheduleReload<commit_after>/*    Copyright 2016-2020 Firewalla Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */


/*
 * 2.2.15 RemoteHost
 *
 * This variable represents the source of inbound IP packets. This will be a wildcard in most cases
 * (i.e. an empty string). NAT vendors are only required to support wildcards. A non-wildcard value
 * will allow for “narrow” port mappings, which may be desirable in some usage scenarios.When
 * RemoteHost is a wildcard, all traffic sent to the ExternalPort on the WAN interface of the
 * gateway is forwarded to the InternalClient on the InternalPort. When RemoteHost is
 * specified as one external IP address as opposed to a wildcard, the NAT will only forward inbound
 * packets from this RemoteHost to the InternalClient, all other packets will be dropped.
 *
 * http://upnp.org/specs/gw/UPnP-gw-WANIPConnection-v1-Service.pdf
 */

'use strict';
const log = require('../net2/logger.js')(__filename);

const Sensor = require('./Sensor.js').Sensor;
const sem = require('./SensorEventManager').getInstance();

const rclient = require('../util/redis_manager.js').getRedisClient()

const UPNP = require('../extension/upnp/upnp.js');
const upnp = new UPNP();

const cfg = require('../net2/config.js');

const sysManager = require('../net2/SysManager.js');
const Message = require('../net2/Message.js');
const Alarm = require('../alarm/Alarm.js');
const AM2 = require('../alarm/AlarmManager2.js');
const am2 = new AM2();
const platform = require('../platform/PlatformLoader.js').getPlatform();
const fs = require('fs');
const exec = require('child-process-promise').exec;

const _ = require('lodash');

const ALARM_UPNP = 'alarm_upnp';

function compareUpnp(a, b) {
  return a.public && b.public &&
    a.private && b.private &&
    a.public.port == b.public.port &&
    a.private.host == b.private.host &&
    a.private.port == b.private.port &&
    a.protocol == b.protocol;
}

class UPNPSensor extends Sensor {
  constructor() {
    super();
    this.upnpLeaseFileWatchers = [];
  }

  isExpired(mapping) {
    const retentionPeriod = this.config.expireInterval || 1800;
    return !mapping.lastSeen || mapping.lastSeen < (Math.floor(Date.now() / 1000) - retentionPeriod) || (mapping.expire && mapping.expire < Math.floor(Date.now() / 1000));
  }

  mergeResults(curMappings, preMappings) {

    curMappings.forEach((mapping) => {
      mapping.expire = (mapping.ttl && mapping.ttl > 0) ? Math.floor(Date.now() / 1000) + mapping.ttl : null;
      mapping.lastSeen = Math.floor(Date.now() / 1000);
    });

    const fullMappings = [...curMappings, ...preMappings];

    const uniqMappings = _.uniqWith(fullMappings, compareUpnp);

    return uniqMappings
      .filter((mapping) => !this.isExpired(mapping));
  }

  scheduleReload() {
    if (this.reloadTask)
      clearTimeout(this.reloadTask);
    this.reloadTask = setTimeout(async () => {
      // UPnP lease file only exists on FireRouter enabled devices
      if (!platform.isFireRouterManaged())
        return;
      for (const watcher of this.upnpLeaseFileWatchers) {
        watcher.close();
      }
      this.upnpLeaseFileWatchers = [];
      const monitoringInterfaces = sysManager.getMonitoringInterfaces();
      for (const iface of monitoringInterfaces) {
        if (iface.name && iface.name.endsWith(":0"))
          continue;
        const leaseFile = `/var/run/upnp.${iface.name}.leases`;
        await exec(`sudo touch ${leaseFile}`).then(() => {
          const watcher = fs.watch(leaseFile, {}, (e, filename) => {
            if (e === "change") {
              log.info(`UPnP lease file ${leaseFile} is changed, schedule checking UPnP leases ...`);
              this.scheduleCheckUPnPLeases();
            }
            if (e === "rename") {
              log.info(`UPnP lease file ${leaseFile} is renamed, schedule reload UPNPSensor ...`);
              this.scheduleReload();
            }
          });
          log.info(`Watching UPnP lease file change on ${leaseFile} ...`);
          this.upnpLeaseFileWatchers.push(watcher);
        }).catch((err) => {
          log.error(`Failed to watch file change ${leaseFile}`, err.message);
        });
      }
      this.scheduleCheckUPnPLeases();
    }, 5000);
  }

  scheduleCheckUPnPLeases() {
    if (this.checkTask)
      clearTimeout(this.checkTask);
{'fraction_non_alphanumeric': 0.1309028130395044, 'fraction_numerical': 0.009749162181375039, 'mean_word_length': 3.057684384013185, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1508514', 'n_tokens_mistral': 3002, 'n_tokens_neox': 2912, 'n_words': 841}