<commit_before>/**
 * MongoDB storage implementation.
 */

'use strict';

var mongodb = require('mongodb'),
    BSON = mongodb.BSONPure,
    config = diana.getConfig().mongodb;

exports.createInstance = function() {
  return new MongoDb();
};

var MongoDb = function() {};

// Db instance.
MongoDb.prototype.link = null;

// Read from app/config.js.
MongoDb.prototype.eventCollection = null;

/**
 * Post-process events found via findOne(), find(), etc.
 *
 * @param docs {Array|Object} Query result(s).
 * @param options {Object} Query options used to produce 'docs'.
 * @return {Object}
 * - info {Object}
 *   prevPage {Boolean}
 *   nextPage {Boolean}
 * - docs {Array}
 */
MongoDb.prototype.eventPostFind = function(docs, options) {
  docs = _.isArray(docs) ? docs : [docs];
  options = options || {};
  var post = {
    info: {
      prevPage: docs && options.skip > 0,
      nextPage: docs && (docs.length == options.limit)
    },
    docs: docs
  };
  if (post.docs) {
    if (post.info.nextPage) { // Discard next-page hint doc.
      post.docs.pop();
    }
    post.docs = _.map(post.docs, function(doc) {
      doc.time = doc.time.getTime();
      return doc;
    });
  }
  return post;
};

/**
 * Extract find()-compatible options from an object containing app-specific
 * keys, e.g. 'sort-dir' and 'sort-attr' which are converted into {sort: {...}}.
 *
 * @param params {Object}
 * @return {Object}
 */
MongoDb.prototype.extractSortOptions = function(params) {
  var options = {};
  if (params['sort-attr']) {
    if ('desc' == params['sort-dir']) {
      options.sort = [[params['sort-attr'], 'desc']];
      delete params['sort-dir'];
    } else if ('asc' == params['sort-dir']) {
      options.sort = [[params['sort-attr'], 'asc']];
      delete params['sort-dir'];
    }
    delete params['sort-attr'];

    if ('time' == options.sort[0][0]) {
      options.sort.push(['_id', options.sort[0][1]]);
    }
  }
  if (params.limit) {
    options.limit = Math.min(parseInt(params.limit, 10), config.maxResultSize);
    delete params.limit;
  } else {
    options.limit = config.maxResultSize;
  }
  if (params.skip) {
    options.skip = parseInt(params.skip, 10);
    delete params.skip;
  }
  options.limit += 1; // For next-page detection.
  return options;
};

/**
 * Extract find()-compatible filters from an object containing app-specific
 * keys, e.g. 'time-gte' which is converted into {time: {$gte: ...}}.
 *
 * @param params {Object} Modified in-place with extracted options.
 */
MongoDb.prototype.extractFilterOptions = function(params) {
  _.each(params, function(value, key) {
    var matches = null;
    if ((matches = key.match(/^(.*)-(gte|gt|lte|lt|ne)$/))) {
      if ('time' == matches[1]) {
        value = new Date(parseInt(value, 10));
      }
      if (!params[matches[1]]) {
        params[matches[1]] = {};
      }
      params[matches[1]]['$' + matches[2]] = value;
      delete params[key];
    } else if ('_id' == key && _.isString(value)) {
      params[key] = new BSON.ObjectID(value);
    } else if ('_id' == key && _.isObject(value)) {
      _.each(value, function(v, comparison) {
        value[comparison] = new BSON.ObjectID(v);
      });
    }
  });
};

/**
 * Connect to the DB based on config.js. Reuse a link if available.
 *
 * @param error {Function} Fired after error.
 * @param success {Function} Fired after success.
 */
MongoDb.prototype.dbConnectAndOpen = function(error, success) {
  if (this.link) {
    success(null, this.link);
  } else {
    this.eventCollection = config.eventCollection;
    this.link = new mongodb.Db(
      config.db,
      new mongodb.Server(config.host, config.port, {})
    );<commit_msg>Fix: _id values were not converted to strings.<commit_after>/**
 * MongoDB storage implementation.
 */

'use strict';

var mongodb = require('mongodb'),
    BSON = mongodb.BSONPure,
    config = diana.getConfig().mongodb;

exports.createInstance = function() {
  return new MongoDb();
};

var MongoDb = function() {};

// Db instance.
MongoDb.prototype.link = null;

// Read from app/config.js.
MongoDb.prototype.eventCollection = null;

/**
 * Post-process events found via findOne(), find(), etc.
 *
 * @param docs {Array|Object} Query result(s).
 * @param options {Object} Query options used to produce 'docs'.
 * @return {Object}
 * - info {Object}
 *   prevPage {Boolean}
 *   nextPage {Boolean}
 * - docs {Array}
 */
MongoDb.prototype.eventPostFind = function(docs, options) {
  docs = _.isArray(docs) ? docs : [docs];
  options = options || {};
  var post = {
    info: {
      prevPage: docs && options.skip > 0,
      nextPage: docs && (docs.length == options.limit)
    },
    docs: docs
  };
  if (post.docs) {
    if (post.info.nextPage) { // Discard next-page hint doc.
      post.docs.pop();
    }
    post.docs = _.map(post.docs, function(doc) {
      doc.time = doc.time.getTime();
      return doc;
    });
  }
  return post;
};

/**
 * Extract find()-compatible options from an object containing app-specific
 * keys, e.g. 'sort-dir' and 'sort-attr' which are converted into {sort: {...}}.
 *
 * @param params {Object}
 * @return {Object}
 */
MongoDb.prototype.extractSortOptions = function(params) {
  var options = {};
  if (params['sort-attr']) {
    if ('desc' == params['sort-dir']) {
      options.sort = [[params['sort-attr'], 'desc']];
      delete params['sort-dir'];
    } else if ('asc' == params['sort-dir']) {
      options.sort = [[params['sort-attr'], 'asc']];
      delete params['sort-dir'];
    }
    delete params['sort-attr'];

    if ('time' == options.sort[0][0]) {
      options.sort.push(['_id', options.sort[0][1]]);
    }
  }
  if (params.limit) {
    options.limit = Math.min(parseInt(params.limit, 10), config.maxResultSize);
    delete params.limit;
  } else {
    options.limit = config.maxResultSize;
  }
  if (params.skip) {
    options.skip = parseInt(params.skip, 10);
    delete params.skip;
  }
  options.limit += 1; // For next-page detection.
  return options;
};

/**
 * Extract find()-compatible filters from an object containing app-specific
 * keys, e.g. 'time-gte' which is converted into {time: {$gte: ...}}.
 *
 * @param params {Object} Modified in-place with extracted options.
 */
MongoDb.prototype.extractFilterOptions = function(params) {
  _.each(params, function(value, key) {
    var matches = null;
    if ((matches = key.match(/^(.*)-(gte|gt|lte|lt|ne)$/))) {
      if ('time' == matches[1]) {
        value = new Date(parseInt(value, 10));
      }
      if (!params[matches[1]]) {
        params[matches[1]] = {};
      }
      params[matches[1]]['$' + matches[2]] = value;
      delete params[key];
    } else if ('_id' == key && _.isString(value)) {
      params[key] = new BSON.ObjectID(value);
    } else if ('_id' == key && _.isObject(value)) {
      _.each(value, function(v, comparison) {
        value[comparison] = new BSON.ObjectID(v.toString());
      });
    }
  });
};

/**
 * Connect to the DB based on config.js. Reuse a link if available.
 *
 * @param error {Function} Fired after error.
 * @param success {Function} Fired after success.
 */
MongoDb.prototype.dbConnectAndOpen = function(error, success) {
  if (this.link) {
    success(null, this.link);
  } else {
    this.eventCollection = config.eventCollection;
    this.link = new mongodb.Db(
      config.db,
      new mongodb.Server(config.host, config.port, {})
    );
{'fraction_non_alphanumeric': 0.1755549539794261, 'fraction_numerical': 0.0046020573903627505, 'mean_word_length': 2.9768568353067817, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4000149', 'n_tokens_mistral': 2476, 'n_tokens_neox': 2419, 'n_words': 587}