<commit_before>###
# Copyright 2013-2016  <NAME>
#
# GenieACS is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# GenieACS is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
###

config = require './config'
mongodb = require 'mongodb'
redis = require('redis')
common = require './common'

redisClient = null
tasksCollection = null
devicesCollection = null
presetsCollection = null
objectsCollection = null
provisionsCollection = null
virtualParametersCollection = null
faultsCollection = null
filesCollection = null
operationsCollection = null


connect = (callback) ->
  callbackCounter = 10
  mongodb.MongoClient.connect(config.get('MONGODB_CONNECTION_URL'), {db:{w:1},server:{autoReconnect:true}}, (err, db) ->
    return callback(err) if err
    exports.mongoDb = db
    db.collection('tasks', (err, collection) ->
      exports.tasksCollection = tasksCollection = collection
      collection?.ensureIndex({device: 1, timestamp: 1}, (err) ->
      )

      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('devices', (err, collection) ->
      exports.devicesCollection  = devicesCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('presets', (err, collection) ->
      exports.presetsCollection = presetsCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('objects', (err, collection) ->
      exports.objectsCollection = objectsCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('fs.files', (err, collection) ->
      exports.filesCollection = filesCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('provisions', (err, collection) ->
      exports.provisionsCollection = provisionsCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('virtualParameters', (err, collection) ->
      exports.virtualParametersCollection = virtualParametersCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('faults', (err, collection) ->
      exports.faultsCollection = faultsCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('operations', (err, collection) ->
      exports.operationsCollection = operationsCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    exports.redisClient = redisClient = redis.createClient(config.get('REDIS_PORT'), config.get('REDIS_HOST'))
    redisClient.select(config.get('REDIS_DB'), (err) ->
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )
  )


disconnect = () ->
  exports.mongoDb?.close()
  redisClient?.quit()


# Optimize projection by removing overlaps
# This modifies given object and returns it
optimizeProjection = (obj) ->
  keys = Object.keys(obj).sort()
  return if keys.length <= 1
  i = 1
  while i < keys.length
    a = keys[i-1]
    b = keys[i]
    if b.startsWith(a)
      if b.charAt(a.length) == '.' or b.charAt(a.length - 1) == '.'
        delete obj[b]
        keys.splice(i--, 1)
    ++ i
  return obj


fetchDevice = (id, timestamp, patterns, callback) ->
  MAX_DEPTH = config.get('MAX_DEPTH', id)

  projection = {_id: 1}
  projectionTree = {}
  func = (path, pats, projTree) ->
    children = {}
    for pat in pats
      fragment = pat[0][path.length] or '*'
      load = 0
      if fragment == '*'
        projection[path.join('.')] = 1
        return

      projTree[fragment] ?= {}

      if pat[1] & (1 << path.length)
        projection[path.concat('_timestamp').join('.')] = 1
        projection[path.concat([fragment, '_timestamp']).join('.')] = 1
        projection[path.concat([fragment, '_value']).join('.')] = 1
        projection[path.concat([fragment, '_type']).join('.')] = 1
        projection[path.concat([fragment, '_object']).join('.')] = 1
        projection[path.concat([fragment, '_writable']).join('.')] = 1
        projection[path.concat([fragment, '_orig']).join('.')] = 1

      if pat[1] >> (path.length + 1)
        children[fragment] ?= []
        children[fragment].push(pat)

    for k, v of children
      func(path.concat(k), v, projTree[k])

  func([], patterns, projectionTree)

  delete projectionTree['DeviceID']
  delete projectionTree['Events']
  delete projectionTree['Tags']

  if projection['']?
    proj = {}
  else
    proj = JSON.parse(JSON.stringify(projection))

  for k, v of proj
    if k == '' or k == 'Events' or k.startsWith('Events.')
      if k.length < 7 or k.startsWith('Events.Inform.') or k == 'Events.Inform'
        proj['_lastInform'] = 1
      if k.length < 7 or k.startsWith('Events.1_BOOT.') or k == 'Events.1_BOOT'
        proj['_lastBoot'] = 1
      if k.length < 7 or k.startsWith('Events.0_BOOTSTRAP.') or k == 'Events.0_BOOTSTRAP'
        proj['_lastBootstrap'] = 1
      if k.length < 7 or k.startsWith('Events.Registered.') or k == 'Events.Registered'
        proj['_registered'] = 1
      delete proj[k] if k != ''

    if k == '' or k == 'DeviceID' or k.startsWith('DeviceID.')
      if k.length < 9 or k.startsWith('DeviceID.ID.') or k == 'DeviceID.ID'
        proj['_id'] = 1
      if k.length < 9 or k.startsWith('DeviceID.Manufacturer.') or k == 'DeviceID.DeManufacturer'
        proj['_deviceId._Manufacturer'] = 1
      if k.length < 9 or k.startsWith('DeviceID.ProductClass.') or k == 'DeviceID.ProductClass'
        proj['_deviceId._ProductClass'] = 1
      if k.length < 9 or k.startsWith('DeviceID.OUI.') or k == 'DeviceID.OUI'
        proj['_deviceId._OUI'] = 1
      if k.length < 9 or k.startsWith('DeviceID.SerialNumber.') or k == 'DeviceID.SerialNumber'
        proj['_deviceId._SerialNumber'] = 1
      if k.length < 9
        proj['_deviceId'] = 1
      delete proj[k] if k != ''

    if k == '' or k == 'Tags' or k.startsWith('Tags.')
      proj['_tags'] = 1
      delete proj[k] if k != ''

  optimizeProjection(proj)

  res = []
  loaded = []

  devicesCollection.findOne({'_id' : id}, proj, (err, device) ->
    return callback(err) if err or not device?

    storeParams = (obj, path, timestamp, descendantsFetched, projTree) ->
      if descendantsFetched
        thisFetched = true
      else
        if projection[path.join('.')]
          descendantsFetched = true
          loaded.push([path, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
        if projection[path.concat('_writable').join('.')]
          loaded.push([path, 1 << (path.length - 1)])
          thisFetched = true

      if obj['_timestamp']?
        obj['_timestamp'] = +obj['_timestamp']

      if thisFetched
        attrs = {}

        t = if obj['_timestamp'] > timestamp then obj['_timestamp'] else timestamp

        if obj['_value']?
          attrs.value = [obj['_timestamp'] or 1, [obj['_value'], obj['_type']]]
          attrs.value[1][0] = +attrs.value[1][0] if obj['_type'] == 'xsd:dateTime'
          obj['_object'] = false

        if obj['_writable']?
          attrs.writable = [timestamp, if obj['_writable'] then 1 else 0]

        if obj['_object']?
          attrs.object = [t, if obj['_object'] then 1 else 0]

        res.push([path, t, attrs])

      for k, v of obj
        if k[0] != '_'
          kk = k
          obj['_object'] = true
          storeParams(v, path.concat(k), obj['_timestamp'] or 1, descendantsFetched, projTree?[k])
          delete projTree[kk] if projTree

      if obj['_object']
        if descendantsFetched
          res.push([path.concat('*'), obj['_timestamp']]) if obj['_timestamp']
        else
          for k, v of projTree
            p = path.concat(k)
            res.push([p, +(obj['_timestamp'] ? 1)])
            loaded.push([p, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])

    for k, v of device
      switch k
        when '_lastInform'
          res.push([['Events'], timestamp,
            {object: [timestamp, 1], writable: [timestamp, 0]}])
          res.push([['Events', 'Inform'], +v,
            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
          delete device[k]
        when '_lastInform'
          res.push([['Events'], timestamp,
            {object: [timestamp, 1], writable: [timestamp, 0]}])
          res.push([['Events', '1_BOOT'], +v,
            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
          delete device[k]
        when '_lastBootstrap'<commit_msg>Fix bug in fetching 1_BOOT event from DB<commit_after>###
# Copyright 2013-2016  <NAME>
#
# GenieACS is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# GenieACS is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with GenieACS.  If not, see <http://www.gnu.org/licenses/>.
###

config = require './config'
mongodb = require 'mongodb'
redis = require('redis')
common = require './common'

redisClient = null
tasksCollection = null
devicesCollection = null
presetsCollection = null
objectsCollection = null
provisionsCollection = null
virtualParametersCollection = null
faultsCollection = null
filesCollection = null
operationsCollection = null


connect = (callback) ->
  callbackCounter = 10
  mongodb.MongoClient.connect(config.get('MONGODB_CONNECTION_URL'), {db:{w:1},server:{autoReconnect:true}}, (err, db) ->
    return callback(err) if err
    exports.mongoDb = db
    db.collection('tasks', (err, collection) ->
      exports.tasksCollection = tasksCollection = collection
      collection?.ensureIndex({device: 1, timestamp: 1}, (err) ->
      )

      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('devices', (err, collection) ->
      exports.devicesCollection  = devicesCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('presets', (err, collection) ->
      exports.presetsCollection = presetsCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('objects', (err, collection) ->
      exports.objectsCollection = objectsCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('fs.files', (err, collection) ->
      exports.filesCollection = filesCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('provisions', (err, collection) ->
      exports.provisionsCollection = provisionsCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('virtualParameters', (err, collection) ->
      exports.virtualParametersCollection = virtualParametersCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('faults', (err, collection) ->
      exports.faultsCollection = faultsCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    db.collection('operations', (err, collection) ->
      exports.operationsCollection = operationsCollection = collection
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )

    exports.redisClient = redisClient = redis.createClient(config.get('REDIS_PORT'), config.get('REDIS_HOST'))
    redisClient.select(config.get('REDIS_DB'), (err) ->
      if --callbackCounter == 0 or err
        callbackCounter = 0
        return callback(err)
    )
  )


disconnect = () ->
  exports.mongoDb?.close()
  redisClient?.quit()


# Optimize projection by removing overlaps
# This modifies given object and returns it
optimizeProjection = (obj) ->
  keys = Object.keys(obj).sort()
  return if keys.length <= 1
  i = 1
  while i < keys.length
    a = keys[i-1]
    b = keys[i]
    if b.startsWith(a)
      if b.charAt(a.length) == '.' or b.charAt(a.length - 1) == '.'
        delete obj[b]
        keys.splice(i--, 1)
    ++ i
  return obj


fetchDevice = (id, timestamp, patterns, callback) ->
  MAX_DEPTH = config.get('MAX_DEPTH', id)

  projection = {_id: 1}
  projectionTree = {}
  func = (path, pats, projTree) ->
    children = {}
    for pat in pats
      fragment = pat[0][path.length] or '*'
      load = 0
      if fragment == '*'
        projection[path.join('.')] = 1
        return

      projTree[fragment] ?= {}

      if pat[1] & (1 << path.length)
        projection[path.concat('_timestamp').join('.')] = 1
        projection[path.concat([fragment, '_timestamp']).join('.')] = 1
        projection[path.concat([fragment, '_value']).join('.')] = 1
        projection[path.concat([fragment, '_type']).join('.')] = 1
        projection[path.concat([fragment, '_object']).join('.')] = 1
        projection[path.concat([fragment, '_writable']).join('.')] = 1
        projection[path.concat([fragment, '_orig']).join('.')] = 1

      if pat[1] >> (path.length + 1)
        children[fragment] ?= []
        children[fragment].push(pat)

    for k, v of children
      func(path.concat(k), v, projTree[k])

  func([], patterns, projectionTree)

  delete projectionTree['DeviceID']
  delete projectionTree['Events']
  delete projectionTree['Tags']

  if projection['']?
    proj = {}
  else
    proj = JSON.parse(JSON.stringify(projection))

  for k, v of proj
    if k == '' or k == 'Events' or k.startsWith('Events.')
      if k.length < 7 or k.startsWith('Events.Inform.') or k == 'Events.Inform'
        proj['_lastInform'] = 1
      if k.length < 7 or k.startsWith('Events.1_BOOT.') or k == 'Events.1_BOOT'
        proj['_lastBoot'] = 1
      if k.length < 7 or k.startsWith('Events.0_BOOTSTRAP.') or k == 'Events.0_BOOTSTRAP'
        proj['_lastBootstrap'] = 1
      if k.length < 7 or k.startsWith('Events.Registered.') or k == 'Events.Registered'
        proj['_registered'] = 1
      delete proj[k] if k != ''

    if k == '' or k == 'DeviceID' or k.startsWith('DeviceID.')
      if k.length < 9 or k.startsWith('DeviceID.ID.') or k == 'DeviceID.ID'
        proj['_id'] = 1
      if k.length < 9 or k.startsWith('DeviceID.Manufacturer.') or k == 'DeviceID.DeManufacturer'
        proj['_deviceId._Manufacturer'] = 1
      if k.length < 9 or k.startsWith('DeviceID.ProductClass.') or k == 'DeviceID.ProductClass'
        proj['_deviceId._ProductClass'] = 1
      if k.length < 9 or k.startsWith('DeviceID.OUI.') or k == 'DeviceID.OUI'
        proj['_deviceId._OUI'] = 1
      if k.length < 9 or k.startsWith('DeviceID.SerialNumber.') or k == 'DeviceID.SerialNumber'
        proj['_deviceId._SerialNumber'] = 1
      if k.length < 9
        proj['_deviceId'] = 1
      delete proj[k] if k != ''

    if k == '' or k == 'Tags' or k.startsWith('Tags.')
      proj['_tags'] = 1
      delete proj[k] if k != ''

  optimizeProjection(proj)

  res = []
  loaded = []

  devicesCollection.findOne({'_id' : id}, proj, (err, device) ->
    return callback(err) if err or not device?

    storeParams = (obj, path, timestamp, descendantsFetched, projTree) ->
      if descendantsFetched
        thisFetched = true
      else
        if projection[path.join('.')]
          descendantsFetched = true
          loaded.push([path, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])
        if projection[path.concat('_writable').join('.')]
          loaded.push([path, 1 << (path.length - 1)])
          thisFetched = true

      if obj['_timestamp']?
        obj['_timestamp'] = +obj['_timestamp']

      if thisFetched
        attrs = {}

        t = if obj['_timestamp'] > timestamp then obj['_timestamp'] else timestamp

        if obj['_value']?
          attrs.value = [obj['_timestamp'] or 1, [obj['_value'], obj['_type']]]
          attrs.value[1][0] = +attrs.value[1][0] if obj['_type'] == 'xsd:dateTime'
          obj['_object'] = false

        if obj['_writable']?
          attrs.writable = [timestamp, if obj['_writable'] then 1 else 0]

        if obj['_object']?
          attrs.object = [t, if obj['_object'] then 1 else 0]

        res.push([path, t, attrs])

      for k, v of obj
        if k[0] != '_'
          kk = k
          obj['_object'] = true
          storeParams(v, path.concat(k), obj['_timestamp'] or 1, descendantsFetched, projTree?[k])
          delete projTree[kk] if projTree

      if obj['_object']
        if descendantsFetched
          res.push([path.concat('*'), obj['_timestamp']]) if obj['_timestamp']
        else
          for k, v of projTree
            p = path.concat(k)
            res.push([p, +(obj['_timestamp'] ? 1)])
            loaded.push([p, ((1 << path.length) - 1) ^ ((1 << MAX_DEPTH) - 1)])

    for k, v of device
      switch k
        when '_lastInform'
          res.push([['Events'], timestamp,
            {object: [timestamp, 1], writable: [timestamp, 0]}])
          res.push([['Events', 'Inform'], +v,
            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
          delete device[k]
        when '_lastBoot'
          res.push([['Events'], timestamp,
            {object: [timestamp, 1], writable: [timestamp, 0]}])
          res.push([['Events', '1_BOOT'], +v,
            {object: [+v, 0], writable: [+v, 0], value: [+v, [+v, 'xsd:dateTime']]}])
          delete device[k]
        when '_lastBootstrap'
{'fraction_non_alphanumeric': 0.14474592470663197, 'fraction_numerical': 0.011734720968512717, 'mean_word_length': 2.6886016451233843, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2585909', 'n_tokens_mistral': 6032, 'n_tokens_neox': 5730, 'n_words': 1554}