<commit_before>
/**
 * Created by <NAME> on 04/01/2017.
 */

'use strict';

let instance = null;
let log = null;

const util = require('util');
const spawn = require('child_process').spawn
const f = require('../../net2/Firewalla.js');
const ip = require('ip');
const userID = f.getUserID();
const childProcess = require('child_process');
const execAsync = util.promisify(childProcess.exec);
const Promise = require('bluebird');
const rclient = require('../../util/redis_manager.js').getRedisClient()
const fs = Promise.promisifyAll(require("fs"))

const FILTER_DIR = f.getUserConfigFolder() + "/dns";

const FILTER_FILE = {
  adblock: FILTER_DIR + "/adblock_filter.conf",
  adblockTmp: FILTER_DIR + "/adblock_filter.conf.tmp",

  family: FILTER_DIR + "/family_filter.conf",
  familyTmp: FILTER_DIR + "/family_filter.conf.tmp",

  policy: FILTER_DIR + "/policy_filter.conf"
}

const policyFilterFile = FILTER_DIR + "/policy_filter.conf";
const familyFilterFile = FILTER_DIR + "/family_filter.conf";

const SysManager = require('../../net2/SysManager');
const sysManager = new SysManager();

const fConfig = require('../../net2/config.js').getConfig();

const bone = require("../../lib/Bone.js");

const iptables = require('../../net2/Iptables');
const ip6tables = require('../../net2/Ip6tables.js')

const networkTool = require('../../net2/NetworkTool')();

const dnsmasqBinary = __dirname + "/dnsmasq";
const pidFile = f.getRuntimeInfoFolder() + "/dnsmasq.pid";
const altPidFile = f.getRuntimeInfoFolder() + "/dnsmasq-alt.pid";
const startScriptFile = __dirname + "/dnsmasq.sh";

const configFile = __dirname + "/dnsmasq.conf";
const altConfigFile = __dirname + "/dnsmasq-alt.conf";

const hostsFile = f.getRuntimeInfoFolder() + "/dnsmasq-hosts";
const altHostsFile = f.getRuntimeInfoFolder() + "/dnsmasq-alt-hosts";

const resolvFile = f.getRuntimeInfoFolder() + "/dnsmasq.resolv.conf";

let defaultNameServers = {};
let upstreamDNS = null;

let FILTER_EXPIRE_TIME = 86400 * 1000;

const BLACK_HOLE_IP = "198.51.100.99"
const BLUE_HOLE_IP = "198.51.100.100"

let DEFAULT_DNS_SERVER = (fConfig.dns && fConfig.dns.defaultDNSServer) || "8.8.8.8";

let RELOAD_INTERVAL = 3600 * 24 * 1000; // one day

let statusCheckTimer = null;

module.exports = class DNSMASQ {
  constructor(loglevel) {
    if (instance == null) {
      log = require("../../net2/logger.js")("dnsmasq", loglevel);

      instance = this;

      this.dhcpMode = false;
      this.minReloadTime = new Date() / 1000;
      this.deleteInProgress = false;
      this.shouldStart = false;
      this.needRestart = null;
      this.needWriteHostsFile = null;
      this.failCount = 0 // this is used to track how many dnsmasq status check fails in a row

      this.hashTypes = {
        adblock: 'ads',
        family: 'family'
      };

      this.state = {
        adblock: undefined,
        family: undefined
      };

      this.nextState = {
        adblock: undefined,
        family: undefined
      };

      this.reloadCount = {
        adblock: 0,
        family: 0
      };

      this.nextReloadFilter = {
        adblock: [],
        family: []
      }

      this.counter = {
        reloadDnsmasq: 0,
        writeHostsFile: 0,
        restart: 0
      }

      process.on('exit', () => {
        this.shouldStart = false;
        this.stop();
      });

      setInterval(() => {
        this.checkIfRestartNeeded()
      }, 10 * 1000) // every 10 seconds

      setInterval(() => {
        this.checkIfWriteHostsFile();
      }, 10 * 1000);
    }

    return instance;
  }

  async install() {
    let install_cmd = util.format('cd %s; bash ./install.sh', __dirname);
    try {
      await execAsync(install_cmd);
      log.info("DNSMASQ:INSTALL:Success", "Dnsmasq is installed successfully");
    } catch (err) {
      log.error("DNSMASQ:INSTALL:Error", "Failed to execute script install.sh", err);
      throw err;
    }
  }

  async uninstall() {
    // TODO
  }

  // in format 127.0.0.1#5353
  async setUpstreamDNS(dns) {
    if(dns === upstreamDNS) {
      log.info("upstreamdns is same as dns, ignored. (" + dns + ")");
      return;
    }

    log.info("upstream dns is set to", dns);
    upstreamDNS = dns;

    let enabled = await this.checkStatus();
    
    if(enabled) {
      try {
        await this.start(false);
        log.info("dnsmasq is restarted to apply new upstream dns");
      } catch (err) {
        log.error("Failed to restart dnsmasq to apply new upstream dns");
      }
    } else {
      // do nothing if it is not enabled
    }
  }

  async updateResolvConf() {
    let nameservers = this.getAllDefaultNameServers()
    if (!nameservers || nameservers.length === 0) {
      nameservers = sysManager.myDNS();
    }

    if (!nameservers || nameservers.length === 0) {
      nameservers = [DEFAULT_DNS_SERVER];  // use google dns by default, should not reach this code
    }

    let entries = nameservers.map((nameserver) => "nameserver " + nameserver);
    let config = entries.join('\n');
    config += "\n";
    fs.writeFileSync(resolvFile, config);
  }

  async updateFilter(type, force) {
    let result = await this._updateTmpFilter(type, force);
    if (!result) {
      return;
    }
      
    // needs update
    const filter = FILTER_FILE[type];
    const filterTmp = FILTER_FILE[type + 'Tmp'];

    log.info(`${type} filter file is `, filter);
    log.info(`${type} tmp filter file is `, filterTmp);
    await fs.renameAsync(filterTmp, filter);
  }

  _scheduleNextReload(type, oldNextState, curNextState) {
    if (oldNextState === curNextState) {
      // no need immediate reload when next state not changed during reloading
      this.nextReloadFilter[type].forEach(t => clearTimeout(t));
      this.nextReloadFilter[type].length = 0;
      log.info(`schedule next reload for ${type} in ${RELOAD_INTERVAL/1000}s`);
      this.nextReloadFilter[type].push(setTimeout(this._reloadFilter.bind(this), RELOAD_INTERVAL, type));
    } else {
      log.warn(`${type}'s next state changed from ${oldNextState} to ${curNextState} during reload, will reload again immediately`);
      setImmediate(this._reloadFilter.bind(this), type);
    }
  }
  
  _reloadFilter(type) {
    let preState = this.state[type];
    let nextState = this.nextState[type];
    this.state[type] = nextState;

    log.info(`in reloadFilter(${type}): preState: ${preState}, nextState: ${this.state[type]}, this.reloadCount: ${this.reloadCount[type]++}`);

    if (nextState === true) {
      log.info(`Start to update ${type} filters.`);
      this.updateFilter(type, true)
        .then(result => {
          log.info(`Update ${type} filters successful.`);
          this.reload().then(() => this._scheduleNextReload(type, nextState, this.nextState[type]));
        }).catch(err => {
          log.error(`Update ${type} filters Failed!`, err, {});
        });
    } else {
      if (preState === false && nextState === false) {
        // disabled, no need do anything
        this._scheduleNextReload(type, nextState, this.nextState[type]);
        return;
      }

      log.info(`Start to clean up ${type} filters.`);
      this.cleanUpFilter(type)
        .catch(err => log.error(`Error when clean up ${type} filters`, err, {}))
        .then(() => this.reload().then(() => this._scheduleNextReload(type, nextState, this.nextState[type])));
    }
  }

  controlFilter(type, state) {
    this.nextState[type] = state;
    log.info(`${type} nextState is: ${this.nextState[type]}`);
    if (this.state[type] !== undefined) {
      // already timer running, clear existing ones before trigger next round immediately
      this.nextReloadFilter[type].forEach(t => clearTimeout(t));
      this.nextReloadFilter[type].length = 0;
    }
    setImmediate(this._reloadFilter.bind(this), type);
  }

  async cleanUpFilter(type) {
    const file = FILTER_FILE[type];
    log.info("Clean up filter file:", file);
    try {
      await fs.unlinkAsync(file);
    } catch (err) {
      if (err.code === 'ENOENT') {
        // ignore
        log.info(`Filter file '${file}' not exist, ignore`);
      } else {
        log.error(`Failed to remove filter file: '${file}'`, err, {})
      }
    }
  }

  async addPolicyFilterEntry(domain, options) {
    options = options || {}

    while (this.workingInProgress) {
      await this.delay(1000);  // try again later
    }
    this.workingInProgress = true;

    let entry = null;
    
    if (options.use_blue_hole) {
      entry = util.format("address=/%s/%s\n", domain, BLUE_HOLE_IP)
    } else {
      entry = util.format("address=/%s/%s\n", domain, BLACK_HOLE_IP)
    }

    try {
      await fs.appendFileAsync(policyFilterFile, entry);
    } catch (err) {
      log.error("Failed to write policy data file:", err, {});
    } finally {
      this.workingInProgress = false;
    }
  }

  async removePolicyFilterEntry(domain) {
    while (this.workingInProgress) {
        await this.delay(1000);  // try again later
    }
    this.workingInProgress = true;

    let entry = util.format("address=/%s/%s", domain, BLACK_HOLE_IP);
    try {
      let data = await fs.readFileAsync(policyFilterFile, 'utf8');

      let newData = data.split("\n")
        .filter(line => line !== entry)
        .join("\n");

      await fs.writeFileAsync(policyFilterFile, newData);
    } catch (err) {
      log.error("Failed to write policy data file:", err, {});
    } finally {
      this.workingInProgress = false; // make sure the flag is reset back
    }
  }

  async addPolicyFilterEntries(domains) {
    let entries = domains.map(domain => util.format("address=/%s/%s", domain, BLACK_HOLE_IP));
    let data = entries.join("\n");
    await fs.appendFileAsync(policyFilterFile, data);
  }

  setDefaultNameServers(key, nameservers) {
    defaultNameServers[key] = nameservers;
  }

  unsetDefaultNameServers(key) {
    delete defaultNameServers[key]
  }

  getAllDefaultNameServers() {
    let list = []
    for(let key in defaultNameServers) {
      let l = defaultNameServers[key]
      if(l.constructor.name === 'Array') {
        list.push.apply(list, l)
      }
    }
    return list
  }

  async delay(t) {<commit_msg>support both ip and ip list for nameservers, async file write and better error handling<commit_after>
/**
 * Created by <NAME> on 04/01/2017.
 */

'use strict';

let instance = null;
let log = null;

const util = require('util');
const spawn = require('child_process').spawn
const f = require('../../net2/Firewalla.js');
const ip = require('ip');
const userID = f.getUserID();
const childProcess = require('child_process');
const execAsync = util.promisify(childProcess.exec);
const Promise = require('bluebird');
const rclient = require('../../util/redis_manager.js').getRedisClient();
const fs = Promise.promisifyAll(require("fs"));
const validator = require('validator');

const FILTER_DIR = f.getUserConfigFolder() + "/dns";

const FILTER_FILE = {
  adblock: FILTER_DIR + "/adblock_filter.conf",
  adblockTmp: FILTER_DIR + "/adblock_filter.conf.tmp",

  family: FILTER_DIR + "/family_filter.conf",
  familyTmp: FILTER_DIR + "/family_filter.conf.tmp",

  policy: FILTER_DIR + "/policy_filter.conf"
}

const policyFilterFile = FILTER_DIR + "/policy_filter.conf";
const familyFilterFile = FILTER_DIR + "/family_filter.conf";

const SysManager = require('../../net2/SysManager');
const sysManager = new SysManager();

const fConfig = require('../../net2/config.js').getConfig();

const bone = require("../../lib/Bone.js");

const iptables = require('../../net2/Iptables');
const ip6tables = require('../../net2/Ip6tables.js')

const networkTool = require('../../net2/NetworkTool')();

const dnsmasqBinary = __dirname + "/dnsmasq";
const pidFile = f.getRuntimeInfoFolder() + "/dnsmasq.pid";
const altPidFile = f.getRuntimeInfoFolder() + "/dnsmasq-alt.pid";
const startScriptFile = __dirname + "/dnsmasq.sh";

const configFile = __dirname + "/dnsmasq.conf";
const altConfigFile = __dirname + "/dnsmasq-alt.conf";

const hostsFile = f.getRuntimeInfoFolder() + "/dnsmasq-hosts";
const altHostsFile = f.getRuntimeInfoFolder() + "/dnsmasq-alt-hosts";

const resolvFile = f.getRuntimeInfoFolder() + "/dnsmasq.resolv.conf";

let defaultNameServers = {};
let upstreamDNS = null;

let FILTER_EXPIRE_TIME = 86400 * 1000;

const BLACK_HOLE_IP = "1172.16.31.10"
const BLUE_HOLE_IP = "198.51.100.100"

let DEFAULT_DNS_SERVER = (fConfig.dns && fConfig.dns.defaultDNSServer) || "8.8.8.8";

let RELOAD_INTERVAL = 3600 * 24 * 1000; // one day

let statusCheckTimer = null;

module.exports = class DNSMASQ {
  constructor(loglevel) {
    if (instance == null) {
      log = require("../../net2/logger.js")("dnsmasq", loglevel);

      instance = this;

      this.dhcpMode = false;
      this.minReloadTime = new Date() / 1000;
      this.deleteInProgress = false;
      this.shouldStart = false;
      this.needRestart = null;
      this.needWriteHostsFile = null;
      this.failCount = 0 // this is used to track how many dnsmasq status check fails in a row

      this.hashTypes = {
        adblock: 'ads',
        family: 'family'
      };

      this.state = {
        adblock: undefined,
        family: undefined
      };

      this.nextState = {
        adblock: undefined,
        family: undefined
      };

      this.reloadCount = {
        adblock: 0,
        family: 0
      };

      this.nextReloadFilter = {
        adblock: [],
        family: []
      }

      this.counter = {
        reloadDnsmasq: 0,
        writeHostsFile: 0,
        restart: 0
      }

      process.on('exit', () => {
        this.shouldStart = false;
        this.stop();
      });

      setInterval(() => {
        this.checkIfRestartNeeded()
      }, 10 * 1000) // every 10 seconds

      setInterval(() => {
        this.checkIfWriteHostsFile();
      }, 10 * 1000);
    }

    return instance;
  }

  async install() {
    let install_cmd = util.format('cd %s; bash ./install.sh', __dirname);
    try {
      await execAsync(install_cmd);
      log.info("DNSMASQ:INSTALL:Success", "Dnsmasq is installed successfully");
    } catch (err) {
      log.error("DNSMASQ:INSTALL:Error", "Failed to execute script install.sh", err);
      throw err;
    }
  }

  async uninstall() {
    // TODO
  }

  // in format 127.0.0.1#5353
  async setUpstreamDNS(dns) {
    if(dns === upstreamDNS) {
      log.info("upstreamdns is same as dns, ignored. (" + dns + ")");
      return;
    }

    log.info("upstream dns is set to", dns);
    upstreamDNS = dns;

    let enabled = await this.checkStatus();
    
    if(enabled) {
      try {
        await this.start(false);
        log.info("dnsmasq is restarted to apply new upstream dns");
      } catch (err) {
        log.error("Failed to restart dnsmasq to apply new upstream dns");
      }
    } else {
      // do nothing if it is not enabled
    }
  }

  async updateResolvConf() {
    let nameservers = this.getAllDefaultNameServers()
    if (!nameservers || nameservers.length === 0) {
      nameservers = sysManager.myDNS();
    }

    if (!nameservers || nameservers.length === 0) {
      nameservers = [DEFAULT_DNS_SERVER];  // use google dns by default, should not reach this code
    }

    let entries = nameservers.map(ip => "nameserver " + ip);
    let config = entries.join('\n');
    config += "\n";

    try {
      await fs.writeFileAsync(resolvFile, config);
    } catch (err) {
      log.error("Error when updating resolv.conf:", resolveFile, "error msg:", err.message, {});
      throw err;
    }
  }

  async updateFilter(type, force) {
    let result = await this._updateTmpFilter(type, force);
    if (!result) {
      return;
    }
      
    // needs update
    const filter = FILTER_FILE[type];
    const filterTmp = FILTER_FILE[type + 'Tmp'];

    log.info(`${type} filter file is `, filter);
    log.info(`${type} tmp filter file is `, filterTmp);
    await fs.renameAsync(filterTmp, filter);
  }

  _scheduleNextReload(type, oldNextState, curNextState) {
    if (oldNextState === curNextState) {
      // no need immediate reload when next state not changed during reloading
      this.nextReloadFilter[type].forEach(t => clearTimeout(t));
      this.nextReloadFilter[type].length = 0;
      log.info(`schedule next reload for ${type} in ${RELOAD_INTERVAL/1000}s`);
      this.nextReloadFilter[type].push(setTimeout(this._reloadFilter.bind(this), RELOAD_INTERVAL, type));
    } else {
      log.warn(`${type}'s next state changed from ${oldNextState} to ${curNextState} during reload, will reload again immediately`);
      setImmediate(this._reloadFilter.bind(this), type);
    }
  }
  
  _reloadFilter(type) {
    let preState = this.state[type];
    let nextState = this.nextState[type];
    this.state[type] = nextState;

    log.info(`in reloadFilter(${type}): preState: ${preState}, nextState: ${this.state[type]}, this.reloadCount: ${this.reloadCount[type]++}`);

    if (nextState === true) {
      log.info(`Start to update ${type} filters.`);
      this.updateFilter(type, true)
        .then(result => {
          log.info(`Update ${type} filters successful.`);
          this.reload().then(() => this._scheduleNextReload(type, nextState, this.nextState[type]));
        }).catch(err => {
          log.error(`Update ${type} filters Failed!`, err, {});
        });
    } else {
      if (preState === false && nextState === false) {
        // disabled, no need do anything
        this._scheduleNextReload(type, nextState, this.nextState[type]);
        return;
      }

      log.info(`Start to clean up ${type} filters.`);
      this.cleanUpFilter(type)
        .catch(err => log.error(`Error when clean up ${type} filters`, err, {}))
        .then(() => this.reload().then(() => this._scheduleNextReload(type, nextState, this.nextState[type])));
    }
  }

  controlFilter(type, state) {
    this.nextState[type] = state;
    log.info(`${type} nextState is: ${this.nextState[type]}`);
    if (this.state[type] !== undefined) {
      // already timer running, clear existing ones before trigger next round immediately
      this.nextReloadFilter[type].forEach(t => clearTimeout(t));
      this.nextReloadFilter[type].length = 0;
    }
    setImmediate(this._reloadFilter.bind(this), type);
  }

  async cleanUpFilter(type) {
    const file = FILTER_FILE[type];
    log.info("Clean up filter file:", file);
    try {
      await fs.unlinkAsync(file);
    } catch (err) {
      if (err.code === 'ENOENT') {
        // ignore
        log.info(`Filter file '${file}' not exist, ignore`);
      } else {
        log.error(`Failed to remove filter file: '${file}'`, err, {})
      }
    }
  }

  async addPolicyFilterEntry(domain, options) {
    options = options || {}

    while (this.workingInProgress) {
      await this.delay(1000);  // try again later
    }
    this.workingInProgress = true;

    let entry = null;
    
    if (options.use_blue_hole) {
      entry = util.format("address=/%s/%s\n", domain, BLUE_HOLE_IP)
    } else {
      entry = util.format("address=/%s/%s\n", domain, BLACK_HOLE_IP)
    }

    try {
      await fs.appendFileAsync(policyFilterFile, entry);
    } catch (err) {
      log.error("Failed to write policy data file:", err, {});
    } finally {
      this.workingInProgress = false;
    }
  }

  async removePolicyFilterEntry(domain) {
    while (this.workingInProgress) {
        await this.delay(1000);  // try again later
    }
    this.workingInProgress = true;

    let entry = util.format("address=/%s/%s", domain, BLACK_HOLE_IP);
    try {
      let data = await fs.readFileAsync(policyFilterFile, 'utf8');

      let newData = data.split("\n")
        .filter(line => line !== entry)
        .join("\n");

      await fs.writeFileAsync(policyFilterFile, newData);
    } catch (err) {
      log.error("Failed to write policy data file:", err, {});
    } finally {
      this.workingInProgress = false; // make sure the flag is reset back
    }
  }

  async addPolicyFilterEntries(domains) {
    let entries = domains.map(domain => util.format("address=/%s/%s", domain, BLACK_HOLE_IP));
    let data = entries.join("\n");
    await fs.appendFileAsync(policyFilterFile, data);
  }

  setDefaultNameServers(key, ips) {
    const isIP = (ip) => validator.isIPv4(ip) || validator.isIPv6(ip);
    let _ips;
    if (Array.isArray(ips) {
      _ips = ips.filter(isIP);
    } else {
      if (isIP(ips.toString())) {
        _ips = [ips.toString()];
      } else {
        return;
      }
    }
    defaultNameServers[key] = _ips;
  }

  unsetDefaultNameServers(key) {
    delete defaultNameServers[key]
  }

  getAllDefaultNameServers() {
    let list = []
    for(let key in defaultNameServers) {
      let ips = defaultNameServers[key]
      if(Array.isArray(ips)) {
        Array.prototype.push.apply(list, ips)
      }
    }
    return list
  }

  async delay(t) {
{'fraction_non_alphanumeric': 0.14796654167868475, 'fraction_numerical': 0.010864339967310836, 'mean_word_length': 2.864573657811629, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 8, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '215015', 'n_tokens_mistral': 6966, 'n_tokens_neox': 6697, 'n_words': 1512}