<commit_before>/**
 * Mocks indexing Db adapter
 */
var _ = require('lodash');
var Db = require('backbone-db-local');
var debug = require('../utils/debug')('serverbone:utils:db:mock');
var async = require('async');

var IndexingTestDb = function TestDb() {
  var db = Db.apply(this, arguments);
  this.storage = db.storage;
  this.records = db.records;
};

_.extend(IndexingTestDb.prototype, Db.prototype, {
  addToIndex: function(collection, model, options, cb) {
    var self = this;
    debug.log('add ' + model.id + ' to ' + options.indexKey);
    this.store().getItem(options.indexKey, function(err, items) {
      if (err) return cb(err);
      if (!items) {
        items = [];
      } else {
        items = JSON.parse(items);
      }
      // item was already existing
      if (items.indexOf(model.id) > -1) {
        return cb();
      }
      items.push(model.id);
      self.store().setItem(
        options.indexKey,
        JSON.stringify(items),
        function(err, res) {
          cb(err, model.toJSON(), res);
        });
    });
  },

  readFromIndex: function(collection, options, cb) {
    this.store().getItem(options.indexKey, function(err, ids) {
      ids = _.isString(ids) ? JSON.parse(ids) : ids;
      if (options.limit) {
        ids = ids.splice(options.offset || 0, options.limit);
      }<commit_msg>sortOrder support for mock readFromIndexes<commit_after>/**
 * Mocks indexing Db adapter
 */
var _ = require('lodash');
var Db = require('backbone-db-local');
var debug = require('../utils/debug')('serverbone:utils:db:mock');
var async = require('async');

var IndexingTestDb = function TestDb() {
  var db = Db.apply(this, arguments);
  this.storage = db.storage;
  this.records = db.records;
};

_.extend(IndexingTestDb.prototype, Db.prototype, {
  addToIndex: function(collection, model, options, cb) {
    var self = this;
    debug.log('add ' + model.id + ' to ' + options.indexKey);
    this.store().getItem(options.indexKey, function(err, items) {
      if (err) return cb(err);
      if (!items) {
        items = [];
      } else {
        items = JSON.parse(items);
      }
      // item was already existing
      if (items.indexOf(model.id) > -1) {
        return cb();
      }
      items.push(model.id);
      self.store().setItem(
        options.indexKey,
        JSON.stringify(items),
        function(err, res) {
          cb(err, model.toJSON(), res);
        });
    });
  },

  readFromIndex: function(collection, options, cb) {
    this.store().getItem(options.indexKey, function(err, ids) {
      ids = _.isString(ids) ? JSON.parse(ids) : ids;
      if (options.sortOrder === 1) ids.reverse();
      if (options.limit) {
        ids = ids.splice(options.offset || 0, options.limit);
      }
{'fraction_non_alphanumeric': 0.1636958894143325, 'fraction_numerical': 0.0018188432157148053, 'mean_word_length': 2.696236559139785, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '90105', 'n_tokens_mistral': 889, 'n_tokens_neox': 874, 'n_words': 188}