<commit_before>'use strict'

const Loki = require('lokijs')
const logger = require('../logger')



class ExtendedLoki extends Loki {

  populate(collectionName, data) {

    let collection = this.getCollection(collectionName)
    if(collection === null) {
      logger.trace(`Created collection ${collectionName} in database`)
      collection = this.addCollection(collectionName)
    }

    // data are already in, update based on issue id
    const ids = collection.mapReduce(issue => {
      issue.id
    }, ids => {
      return ids;
    })

    //collection.updateWhere()

    // insert data into database
    collection.insert(data)
    return collection
  }
}


var DB = function(path) {

  // create a Loki.js instance with filesystem storega in particular path
  const db = new ExtendedLoki(path, {
    adapter: new Loki.persistenceAdapters.fs()
  })

  return new Promise((resolve, reject) => {
    db.loadDatabase({}, (err, _) => {
      if(err) {
        logger.trace({path, err}, 'Unable to load database, ignoring')
        return resolve(db)
      }
      logger.trace({path}, 'Database loaded')
      return resolve(db)
    })
  })

}

module.exports = DB
<commit_msg>Updating collection via populate command<commit_after>'use strict'

const Loki = require('lokijs')
const logger = require('../logger')



class ExtendedLoki extends Loki {

  populate(collectionName, data) {

    let collection = this.getCollection(collectionName)
    if(collection === null) {
      logger.trace(`Created collection ${collectionName} in database`)
      collection = this.addCollection(collectionName, { indices: ['id', 'key']})
    }

    // data are already in, update based on issue id
    // update might be done a better way - for now it just deletes all previous entries
    const toBeDeleted = data.reduce((acc, issue) => {
      acc.push(issue.id)
      return acc
    }, [])

    logger.trace(`Going to drop ${toBeDeleted.length} entries from collection ${collectionName} as they have been updated`)
    collection.removeWhere(issue => {
      return toBeDeleted.includes(issue.id)
    })

    // insert data into database
    logger.trace(`Inserting ${data.length} entries into collection ${collectionName}`)
    collection.insert(data)
    logger.trace(`Collection ${collectionName} now contains ${collection.count()} entries`)
    return collection
  }
}


var DB = function(path) {

  // create a Loki.js instance with filesystem storega in particular path
  const db = new ExtendedLoki(path, {
    adapter: new Loki.persistenceAdapters.fs()
  })

  return new Promise((resolve, reject) => {
    db.loadDatabase({}, (err, _) => {
      if(err) {
        logger.trace({path, err}, 'Unable to load database, ignoring')
        return resolve(db)
      }
      logger.trace({path}, 'Database loaded')
      return resolve(db)
    })
  })

}

module.exports = DB

{'fraction_non_alphanumeric': 0.12212943632567849, 'fraction_numerical': 0.0, 'mean_word_length': 3.142651296829971, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2798892', 'n_tokens_mistral': 834, 'n_tokens_neox': 809, 'n_words': 235}