<commit_before>/*    Copyright 2016-2019 Firewalla INC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

'use strict'

let log = require('../net2/logger.js')(__filename, 'info');
let ip = require('ip')


const minimatch = require('minimatch')

const _ = require('lodash')

const validator = require('validator');

const CategoryMatcher = require('./CategoryMatcher');

function arraysEqual(a, b) {
  if (a === b) return true;
  if (a == null || b == null) return false;
  if (a.length != b.length) return false;

  // If you don't care about the order of the elements inside
  // the array, you should sort both arrays here.

  for (var i = 0; i < a.length; ++i) {
    if (a[i] !== b[i]) return false;
  }
  return true;
}

function isJsonString(str) {
  return _.isString(str) && validator.isJSON(str)
}

module.exports = class {
  constructor(raw) {
    const numberKeys = ["timestamp", "expireTs", "idleTs"]
    for (const key in raw) {
      if (isJsonString(raw[key])) {
        // parse will always be successful if passed check
        raw[key] = JSON.parse(raw[key])
      }
      if (numberKeys.includes(key))
        raw[key] = !isNaN(raw[key]) && Number(raw[key]);
      if (key == 'p.tag.ids' && Array.isArray(raw[key])) {
        raw[key] = raw[key].map(String) //Backward compatibility
      }
      if ((/^p\.tag\.ids\.[0-9]+$/).test(key)) {
        log.debug('legacy field found', key, raw[key])
        this['p.tag.ids'] = this['p.tag.ids'] || []
        this['p.tag.ids'].push(raw[key])
        delete raw[key]
      }
    }
    Object.assign(this, raw);

    if (!this.timestamp) this.timestamp = new Date() / 1000;
  }

  getMatchingKeys() {
    let keys = ["cronTime", "duration", "expireTs", "idleTs"]
    for (let k in this) {
      if (k === "type" || k.startsWith("p.") || k.startsWith("e.")) {
        keys.push(k)
      }
    }

    return keys.sort()
  }

  isEqualToException(e) {
    const thisKeys = this.getMatchingKeys()
    const thatKeys = e.getMatchingKeys()

    if (!arraysEqual(thisKeys, thatKeys)) {
      return false
    }
    for (let i in thisKeys) {
      let k = thisKeys[i]
      if (this[k] !== e[k]) {
        return false
      }
    }

    return true
  }

  isSecurityAlarm(alarm) {
    const securityAlarmTypes = [
      "ALARM_SPOOFING_DEVICE",
      "ALARM_BRO_NOTICE",
      "ALARM_INTEL",
      "ALARM_VULNERABILITY"
    ];

    return securityAlarmTypes.includes(alarm.type);
  }

  jsonComparisonMatch(val, val2) {
    if (!isFinite(val2)) return false;
    let comparison = JSON.parse(val);
    if (isFinite(comparison["$gt"])) {
      if (val2 > comparison["$gt"]) {<commit_msg>use lodash's isEqual to compare keys in Exception.isEqualToException<commit_after>/*    Copyright 2016-2019 Firewalla INC
 *
 *    This program is free software: you can redistribute it and/or  modify
 *    it under the terms of the GNU Affero General Public License, version 3,
 *    as published by the Free Software Foundation.
 *
 *    This program is distributed in the hope that it will be useful,
 *    but WITHOUT ANY WARRANTY; without even the implied warranty of
 *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *    GNU Affero General Public License for more details.
 *
 *    You should have received a copy of the GNU Affero General Public License
 *    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 */

'use strict'

let log = require('../net2/logger.js')(__filename, 'info');
let ip = require('ip')


const minimatch = require('minimatch')

const _ = require('lodash')

const validator = require('validator');

const CategoryMatcher = require('./CategoryMatcher');

function arraysEqual(a, b) {
  if (a === b) return true;
  if (a == null || b == null) return false;
  if (a.length != b.length) return false;

  // If you don't care about the order of the elements inside
  // the array, you should sort both arrays here.

  for (var i = 0; i < a.length; ++i) {
    if (a[i] !== b[i]) return false;
  }
  return true;
}

function isJsonString(str) {
  return _.isString(str) && validator.isJSON(str)
}

module.exports = class {
  constructor(raw) {
    const numberKeys = ["timestamp", "expireTs", "idleTs"]
    for (const key in raw) {
      if (isJsonString(raw[key])) {
        // parse will always be successful if passed check
        raw[key] = JSON.parse(raw[key])
      }
      if (numberKeys.includes(key))
        raw[key] = !isNaN(raw[key]) && Number(raw[key]);
      if (key == 'p.tag.ids' && Array.isArray(raw[key])) {
        raw[key] = raw[key].map(String) //Backward compatibility
      }
      if ((/^p\.tag\.ids\.[0-9]+$/).test(key)) {
        log.debug('legacy field found', key, raw[key])
        this['p.tag.ids'] = this['p.tag.ids'] || []
        this['p.tag.ids'].push(raw[key])
        delete raw[key]
      }
    }
    Object.assign(this, raw);

    if (!this.timestamp) this.timestamp = new Date() / 1000;
  }

  getMatchingKeys() {
    let keys = ["cronTime", "duration", "expireTs", "idleTs"]
    for (let k in this) {
      if (k === "type" || k.startsWith("p.") || k.startsWith("e.")) {
        keys.push(k)
      }
    }

    return keys.sort()
  }

  isEqualToException(e) {
    const thisKeys = this.getMatchingKeys()
    const thatKeys = e.getMatchingKeys()

    if (!arraysEqual(thisKeys, thatKeys)) {
      return false
    }
    for (let i in thisKeys) {
      let k = thisKeys[i]
      if (!_.isEqual(this[k], e[k])) {
        return false
      }
    }

    return true
  }

  isSecurityAlarm(alarm) {
    const securityAlarmTypes = [
      "ALARM_SPOOFING_DEVICE",
      "ALARM_BRO_NOTICE",
      "ALARM_INTEL",
      "ALARM_VULNERABILITY"
    ];

    return securityAlarmTypes.includes(alarm.type);
  }

  jsonComparisonMatch(val, val2) {
    if (!isFinite(val2)) return false;
    let comparison = JSON.parse(val);
    if (isFinite(comparison["$gt"])) {
      if (val2 > comparison["$gt"]) {
{'fraction_non_alphanumeric': 0.14443422263109476, 'fraction_numerical': 0.0061330880098129405, 'mean_word_length': 2.7990681421083283, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1731013', 'n_tokens_mistral': 2173, 'n_tokens_neox': 2118, 'n_words': 611}