<commit_before>/*  Copyright (C) 2016 <NAME>
    Copyright (C) 2016 HopGlass Server contributors

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

'use strict'

var fs = require('fs')
var _ = require('lodash')
var async = require('async')

var config = {
  /* eslint-disable quotes */
  ifaces: [
    "bat0"
  ],
  storage: {
    interval: 300,
    file: "./raw.json"
  },
  purge: {
    maxAge: 14,      // in days
    interval: 86400  //  1 day
  }
}

module.exports = function (configData) {
  if (configData.ifaces && configData.ifaces.length > 0)
    delete config.ifaces

  _.merge(config, configData)

  var receiverList = {}
  var raw = {}

  try {
    raw = JSON.parse(fs.readFileSync(config.storage.file, 'utf8'))
  } catch(err) {
    console.log(err)
  }

  require('fs').readdirSync(__dirname + '/receiver').forEach(function(e) {
    var re = /\.js$/
    if (re.test(e))
      receiverList[e.replace(re, '')] = require(__dirname + '/receiver/' + e)(config)
  })

  function getRaw() {
    var now = new Date().getTime()
    if (typeof getRaw.lastUpdate === 'undefined' || now - getRaw.lastUpdate >= 10*1000) { // experimental cache
      getRaw.lastUpdate = now
      _.forEach(receiverList, function(e) {
        if (!e.overwrite) {
          _.assignWith(raw, e.getRaw(), function(objValue, srcValue) {
            if (_.isUndefined(objValue)) {
              return srcValue
            } else {
              var srcDate = new Date(srcValue.lastseen)
              var objDate = new Date(objValue.lastseen)
              if (srcDate.getTime() >= objDate.getTime()) {
                return srcValue
              } else {
                return objValue
              }
            }
          })
        }
      })
    }
    return raw
  }

  function getData(query) {
    var data = getRaw()
    _.forEach(receiverList, function(e) {
      if (e.overwrite) {
        data = _.merge(data, e.getRaw())
      }
    })

    if (typeof query === 'object')
      data = filterData(data, query)
<commit_msg>temp. fix: firstseen was reset on server restart<commit_after>/*  Copyright (C) 2016 <NAME>
    Copyright (C) 2016 HopGlass Server contributors

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as published
    by the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>. */

'use strict'

var fs = require('fs')
var _ = require('lodash')
var async = require('async')

var config = {
  /* eslint-disable quotes */
  ifaces: [
    "bat0"
  ],
  storage: {
    interval: 300,
    file: "./raw.json"
  },
  purge: {
    maxAge: 14,      // in days
    interval: 86400  //  1 day
  }
}

module.exports = function (configData) {
  if (configData.ifaces && configData.ifaces.length > 0)
    delete config.ifaces

  _.merge(config, configData)

  var receiverList = {}
  var raw = {}

  try {
    raw = JSON.parse(fs.readFileSync(config.storage.file, 'utf8'))
  } catch(err) {
    console.log(err)
  }

  require('fs').readdirSync(__dirname + '/receiver').forEach(function(e) {
    var re = /\.js$/
    if (re.test(e))
      receiverList[e.replace(re, '')] = require(__dirname + '/receiver/' + e)(config)
  })

  function getRaw() {
    var now = new Date().getTime()
    if (typeof getRaw.lastUpdate === 'undefined' || now - getRaw.lastUpdate >= 10*1000) { // experimental cache
      getRaw.lastUpdate = now
      _.forEach(receiverList, function(e) {
        if (!e.overwrite) {
          var firstseenTmp = {}
          _.assignWith(raw, e.getRaw(), function(objValue, srcValue, key) {
            firstseenTmp[key] = { "firstseen": _.get(objValue, 'firstseen', srcValue.firstseen) }
            if (_.isUndefined(objValue)) {
              return srcValue
            } else {
              var srcDate = new Date(srcValue.lastseen)
              var objDate = new Date(objValue.lastseen)
              if (srcDate.getTime() >= objDate.getTime()) {
                return srcValue
              } else {
                return objValue
              }
            }
          })
          _.merge(raw, firstseenTmp)
        }
      })
    }
    return raw
  }

  function getData(query) {
    var data = getRaw()
    _.forEach(receiverList, function(e) {
      if (e.overwrite) {
        data = _.merge(data, e.getRaw())
      }
    })

    if (typeof query === 'object')
      data = filterData(data, query)

{'fraction_non_alphanumeric': 0.1171122022185852, 'fraction_numerical': 0.01054737224949991, 'mean_word_length': 2.315250150693189, 'pattern_counts': {'":': 1, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2087146', 'n_tokens_mistral': 1706, 'n_tokens_neox': 1629, 'n_words': 501}