<commit_before>// setup some dependencies
var express  = require('express'),
    cors = require('cors'),
    compression = require('compression'),
    cache    = require('web-cache'),
    validate = require('conform').validate,
    MongoClient  = require('mongodb').MongoClient;

// load settings from config file
var settings = require('./config/settings.json');
var collections = require('./config/collections').collections;
var MongoAPI = require('./config/api').api;

// add mongodb collections to the external services
var services = settings.externalServices;
for (var c in collections) {
  services['/' + c] = collections[c].description;
}

// setup mongodb collections
var databases = {};
for (var coll in collections) {
  var c = collections[coll];
  if (databases.hasOwnProperty(c.dbName)) {
    databases[c.dbName].colls.push(coll);
  }
  else {
    databases[c.dbName] = {
      url: 'mongodb://' + c.host + ':' + c.port + '/' + c.dbName,
      colls: [coll]
    };
  }
}
for (var dbName in databases) {
  (function(url, colls) {
    MongoClient.connect(url, function(err, db) {
      if (err) throw err;
      console.log('connected to ' + url, colls);
      colls.forEach(function (coll) {
        var c = collections[coll];
        c.coll = db.collection(c.collectionName);
      });
    });
  })(databases[dbName].url, databases[dbName].colls);
}

function buildQuery(params, cmd) {
  var qExprs = [];
  if (params.hasOwnProperty('q')) qExprs.push({'$text': {'$search': params['q']}});
  if (params.hasOwnProperty('l')) {
    var a = params['l'].split(':');
    var m = a[0], r = a[1], s = a[2], e = a[3];
    qExprs.push(
      {'location.map':m},
      {'location.region':r},
      {'location.start':{'$lte': +e}},
      {'location.end':{'$gte': +s}});
  }
  for (var p in params) {
    if (!cmd.hasOwnProperty(p)) {
      var o = {};
      o[p] = params[p];
      qExprs.push(o);
    }
  }
  if (qExprs.length > 1) return {'$and': qExprs};
  else if (qExprs.length == 1) return qExprs[0];
  else return {};
}

// the actual mongodb queries for each API command
var MongoCommand = {
  describe: function(c, params, req, res) {
    res.send(c.properties);
  },
  select : function(c, params, req,  res) {
    var query = buildQuery(params, MongoAPI.select.properties);
    var time = process.hrtime();
    c.coll.count(query, function(err, count) {
      if (err) throw err;
      var options = {};<commit_msg>added idList parameter to do this {_id : { : [ list of ids]}}<commit_after>// setup some dependencies
var express  = require('express'),
    cors = require('cors'),
    compression = require('compression'),
    cache    = require('web-cache'),
    validate = require('conform').validate,
    MongoClient  = require('mongodb').MongoClient;

// load settings from config file
var settings = require('./config/settings.json');
var collections = require('./config/collections').collections;
var MongoAPI = require('./config/api').api;

// add mongodb collections to the external services
var services = settings.externalServices;
for (var c in collections) {
  services['/' + c] = collections[c].description;
}

// setup mongodb collections
var databases = {};
for (var coll in collections) {
  var c = collections[coll];
  if (databases.hasOwnProperty(c.dbName)) {
    databases[c.dbName].colls.push(coll);
  }
  else {
    databases[c.dbName] = {
      url: 'mongodb://' + c.host + ':' + c.port + '/' + c.dbName,
      colls: [coll]
    };
  }
}
for (var dbName in databases) {
  (function(url, colls) {
    MongoClient.connect(url, function(err, db) {
      if (err) throw err;
      console.log('connected to ' + url, colls);
      colls.forEach(function (coll) {
        var c = collections[coll];
        c.coll = db.collection(c.collectionName);
      });
    });
  })(databases[dbName].url, databases[dbName].colls);
}

function buildQuery(params, cmd) {
  var qExprs = [];
  if (params.hasOwnProperty('q')) qExprs.push({'$text': {'$search': params['q']}});
  if (params.hasOwnProperty('l')) {
    var a = params['l'].split(':');
    var m = a[0], r = a[1], s = a[2], e = a[3];
    qExprs.push(
      {'location.map':m},
      {'location.region':r},
      {'location.start':{'$lte': +e}},
      {'location.end':{'$gte': +s}});
  }
  for (var p in params) {
    if (!cmd.hasOwnProperty(p)) {
      if (p === 'idList') qExprs.push({'_id': {'$in': params['idList'].split(',').map(function(x) {return +x;})}});
      else {
        var o = {};
        o[p] = params[p];
        qExprs.push(o);
      }
    }
  }
  if (qExprs.length > 1) return {'$and': qExprs};
  else if (qExprs.length == 1) return qExprs[0];
  else return {};
}

// the actual mongodb queries for each API command
var MongoCommand = {
  describe: function(c, params, req, res) {
    res.send(c.properties);
  },
  select : function(c, params, req,  res) {
    var query = buildQuery(params, MongoAPI.select.properties);
    var time = process.hrtime();
    c.coll.count(query, function(err, count) {
      if (err) throw err;
      var options = {};
{'fraction_non_alphanumeric': 0.18374628344895935, 'fraction_numerical': 0.0027750247770069376, 'mean_word_length': 3.1945137157107233, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2685576', 'n_tokens_mistral': 1714, 'n_tokens_neox': 1699, 'n_words': 408}