<commit_before>/*    Copyright 2021-2022 Firewalla Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('../net2/logger.js')(__filename);

const Sensor = require('./Sensor.js').Sensor;

const _ = require('lodash');

const validator = require('validator');

const IntelTool = require('../net2/IntelTool');
const intelTool = new IntelTool();

const rclient = require('../util/redis_manager.js').getRedisClient();
const sclient = require('../util/redis_manager.js').getSubscriptionClient();

const m = require('../extension/metrics/metrics.js');

const flowUtil = require('../net2/FlowUtil');
const f = require('../net2/Firewalla.js');

const userConfigFolder = f.getUserConfigFolder();
const dnsmasqConfigFolder = `${userConfigFolder}/dnsmasq`;

const cc = require('../extension/cloudcache/cloudcache.js');

const fs = require('fs');

const Promise = require('bluebird');
Promise.promisifyAll(fs);

const DNSMASQ = require('../extension/dnsmasq/dnsmasq.js');
const dnsmasq = new DNSMASQ();

const Alarm = require('../alarm/Alarm.js');
const AlarmManager2 = require('../alarm/AlarmManager2.js')
const am2 = new AlarmManager2();
const HostTool = require('../net2/HostTool.js')
const hostTool = new HostTool()
const BF_SERVER_MATCH = "bf_server_match"
const IdentityManager = require('../net2/IdentityManager.js');
const sem = require('../sensor/SensorEventManager.js').getInstance();
const extensionManager = require('./ExtensionManager.js')
const tm = require('../alarm/TrustManager.js');

const bf = require('../extension/bf/bf.js');

// slices a single byte into bits
// assuming only single bytes
const sliceBits = function (b, off, len) {
  const s = 7 - (off + len - 1);

  b = b >>> s;
  return b & ~(0xff << len);
};

const qnameToDomain = (qname) => {
  let domain = '';
  for (let i = 0; i < qname.length; i++) {
    if (qname[i] == 0) {
      //last char chop trailing .
      domain = domain.substring(0, domain.length - 1);
      break;
    }

    const tmpBuf = qname.slice(i + 1, i + qname[i] + 1);
    domain += tmpBuf.toString('binary', 0, tmpBuf.length);
    domain += '.';

    i = i + qname[i];
  }

  return domain;
};

const defaultExpireTime = 48 * 3600; // expire in two days, by default
const allowKey = "dns_proxy:allow_list"; //unused at this moment
const passthroughKey = "dns_proxy:passthrough_list";
const blockKey = "dns_proxy:block_list";
const featureName = "dns_proxy";

class DNSProxyPlugin extends Sensor {
  async run() {
    // invalidate cache keys when starting up
    await rclient.unlinkAsync(allowKey);
    await rclient.unlinkAsync(blockKey);
    await rclient.unlinkAsync(passthroughKey);

    extensionManager.registerExtension(featureName, this, {
      applyPolicy: this.applyDnsProxy
    });
    this.hookFeature(featureName);
  }

  getHashKeyName(item = {}, level = "default") {
    if (!item.count || !item.error || !item.prefix) {
      log.error("Invalid item:", item);
      return null;
    }

    const { count, error, prefix } = item;

    if (level) {
      return `bf:${level}_${prefix}:${count}:${error}`;
    }
    return `bf:${prefix}:${count}:${error}`;
  }

  getFilePath(item = {}) {
    if (!item.count || !item.error || !item.prefix) {
      log.error("Invalid item:", item);
      return null;
    }

    const { count, error, prefix, level } = item;

    if (level) {
      return `${f.getRuntimeInfoFolder()}/${featureName}.${level}_${prefix}.bf.data`;
    }
    return `${f.getRuntimeInfoFolder()}/${featureName}.${prefix}.bf.data`;
  }

  getDnsmasqConfigFile() {
    return `${dnsmasqConfigFolder}/${featureName}.conf`;
  }

  async enableDnsmasqConfig(data) {
    log.info("Enabling dnsmasq config file for dnsproxy...");
    let dnsmasqEntry = "mac-address-tag=%FF:FF:FF:FF:FF:FF$dns_proxy&1\n";
    for (const level in data) {
      const levelData = data[level];
      for (const item of levelData) {
        item.level = level;
        const fp = this.getFilePath(item);
        if (!fp) {
          continue;
        }
        const entry = `server-bf-high=<${fp},${item.count},${item.error}><${allowKey}><${blockKey}><${passthroughKey}>127.0.0.153#59953$dns_proxy\n`;
        dnsmasqEntry += entry;
      }
    }

    await fs.writeFileAsync(this.getDnsmasqConfigFile(), dnsmasqEntry);
    dnsmasq.scheduleRestartDNSService();
  }

  async disableDnsmasqConfig() {
    log.info("Disabling dnsmasq config file for dnsproxy...");
    await fs.unlinkAsync(this.getDnsmasqConfigFile()).catch(() => undefined); // ignore error
    dnsmasq.scheduleRestartDNSService();
  }

  async applyDnsProxy(host, ip, policy) {
    log.info("Applying dns_proxy", ip, policy);
    if (policy) {
      this.dnsProxyData = policy;
    }
    if (!this.dnsProxyData) {
      this.dnsProxyData = {};
      this.dnsProxyData["default"] = this.config.data;
    }

    if (!this.state) {
      log.info("dns_proxy feature is disabled, skip applying policy");
      return;
    }
<commit_msg>use uhigh options in dnsproxy<commit_after>/*    Copyright 2021-2022 Firewalla Inc.
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */
'use strict';

const log = require('../net2/logger.js')(__filename);

const Sensor = require('./Sensor.js').Sensor;

const _ = require('lodash');

const validator = require('validator');

const IntelTool = require('../net2/IntelTool');
const intelTool = new IntelTool();

const rclient = require('../util/redis_manager.js').getRedisClient();
const sclient = require('../util/redis_manager.js').getSubscriptionClient();

const m = require('../extension/metrics/metrics.js');

const flowUtil = require('../net2/FlowUtil');
const f = require('../net2/Firewalla.js');

const userConfigFolder = f.getUserConfigFolder();
const dnsmasqConfigFolder = `${userConfigFolder}/dnsmasq`;

const cc = require('../extension/cloudcache/cloudcache.js');

const fs = require('fs');

const Promise = require('bluebird');
Promise.promisifyAll(fs);

const DNSMASQ = require('../extension/dnsmasq/dnsmasq.js');
const dnsmasq = new DNSMASQ();

const Alarm = require('../alarm/Alarm.js');
const AlarmManager2 = require('../alarm/AlarmManager2.js')
const am2 = new AlarmManager2();
const HostTool = require('../net2/HostTool.js')
const hostTool = new HostTool()
const BF_SERVER_MATCH = "bf_server_match"
const IdentityManager = require('../net2/IdentityManager.js');
const sem = require('../sensor/SensorEventManager.js').getInstance();
const extensionManager = require('./ExtensionManager.js')
const tm = require('../alarm/TrustManager.js');

const bf = require('../extension/bf/bf.js');

// slices a single byte into bits
// assuming only single bytes
const sliceBits = function (b, off, len) {
  const s = 7 - (off + len - 1);

  b = b >>> s;
  return b & ~(0xff << len);
};

const qnameToDomain = (qname) => {
  let domain = '';
  for (let i = 0; i < qname.length; i++) {
    if (qname[i] == 0) {
      //last char chop trailing .
      domain = domain.substring(0, domain.length - 1);
      break;
    }

    const tmpBuf = qname.slice(i + 1, i + qname[i] + 1);
    domain += tmpBuf.toString('binary', 0, tmpBuf.length);
    domain += '.';

    i = i + qname[i];
  }

  return domain;
};

const defaultExpireTime = 48 * 3600; // expire in two days, by default
const allowKey = "dns_proxy:allow_list"; //unused at this moment
const passthroughKey = "dns_proxy:passthrough_list";
const blockKey = "dns_proxy:block_list";
const featureName = "dns_proxy";

class DNSProxyPlugin extends Sensor {
  async run() {
    // invalidate cache keys when starting up
    await rclient.unlinkAsync(allowKey);
    await rclient.unlinkAsync(blockKey);
    await rclient.unlinkAsync(passthroughKey);

    extensionManager.registerExtension(featureName, this, {
      applyPolicy: this.applyDnsProxy
    });
    this.hookFeature(featureName);
  }

  getHashKeyName(item = {}, level = "default") {
    if (!item.count || !item.error || !item.prefix) {
      log.error("Invalid item:", item);
      return null;
    }

    const { count, error, prefix } = item;

    if (level) {
      return `bf:${level}_${prefix}:${count}:${error}`;
    }
    return `bf:${prefix}:${count}:${error}`;
  }

  getFilePath(item = {}) {
    if (!item.count || !item.error || !item.prefix) {
      log.error("Invalid item:", item);
      return null;
    }

    const { count, error, prefix, level } = item;

    if (level) {
      return `${f.getRuntimeInfoFolder()}/${featureName}.${level}_${prefix}.bf.data`;
    }
    return `${f.getRuntimeInfoFolder()}/${featureName}.${prefix}.bf.data`;
  }

  getDnsmasqConfigFile() {
    return `${dnsmasqConfigFolder}/${featureName}.conf`;
  }

  async enableDnsmasqConfig(data) {
    log.info("Enabling dnsmasq config file for dnsproxy...");
    let dnsmasqEntry = "mac-address-tag=%FF:FF:FF:FF:FF:FF$dns_proxy&1\n";
    for (const level in data) {
      const levelData = data[level];
      for (const item of levelData) {
        item.level = level;
        const fp = this.getFilePath(item);
        if (!fp) {
          continue;
        }
        const entry = `server-bf-uhigh=<${fp},${item.count},${item.error}><${allowKey}><${blockKey}><${passthroughKey}>127.0.0.153#59953$dns_proxy\n`;
        dnsmasqEntry += entry;
      }
    }

    await fs.writeFileAsync(this.getDnsmasqConfigFile(), dnsmasqEntry);
    dnsmasq.scheduleRestartDNSService();
  }

  async disableDnsmasqConfig() {
    log.info("Disabling dnsmasq config file for dnsproxy...");
    await fs.unlinkAsync(this.getDnsmasqConfigFile()).catch(() => undefined); // ignore error
    dnsmasq.scheduleRestartDNSService();
  }

  async applyDnsProxy(host, ip, policy) {
    log.info("Applying dns_proxy", ip, policy);
    if (policy) {
      this.dnsProxyData = policy;
    }
    if (!this.dnsProxyData) {
      this.dnsProxyData = {};
      this.dnsProxyData["default"] = this.config.data;
    }

    if (!this.state) {
      log.info("dns_proxy feature is disabled, skip applying policy");
      return;
    }

{'fraction_non_alphanumeric': 0.1526320509371357, 'fraction_numerical': 0.008788449466415568, 'mean_word_length': 3.4697394789579157, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7625862', 'n_tokens_mistral': 3707, 'n_tokens_neox': 3616, 'n_words': 880}