<commit_before>'use strict';

var Arangojs = require('arangojs');
var aql = Arangojs.aql;
var util = require('util');
var slug = require('slug');
var _ = require('lodash');

const DB_SYSTEM = '_system';

/**
 * ArangoDB provider
 * @param {Object} conn Arangojs database connection.
 */
function Arango(conn) {
  this._db = conn;
}

function* query(db, c, q) {
  try {
    return yield db.query(q);
  } catch (err) {
    if (err.message && err.message.startsWith('collection not found')) {
      yield c.create();
    } else {
      throw err;
    }
  }
  return yield db.query(q);
}

function idToKey(id) {
  return slug(id, {
    replacement: '_'
  });
}

function ensureKey(document) {
  if (document._key) {
    return document;
  }
  let id = document.id;
  if (id) {
    document._key = idToKey(id);
  }
  return document;
}

function sanitizeFields(document) {
  delete document._id;
  delete document._key;
  delete document._rev;
  return document;
}

function buildFilter(filter, operator) {
  filter = ensureKey(filter);
  if (!operator) {
    operator = '&&';
  }
  let filterStr = '';
  _.forIn(filter, function(value, key) {
    switch (key) {
    case '$or':
      filterStr += buildFilter(value, '||');
    case '$and':
      filterStr += buildFilter(value, '&&');
    default:
      filterStr += util.format(' node.%s == %s %s', value, key, operator)
    }
  });
  filterStr = _.trimEnd(filterStr, operator);
  return filterStr;
}

/**
 * Inserts documents into database.
 * @param  {String} collection Collection name
 * @param  {Object|array.Object} documents  A single or multiple documents.
 */
Arango.prototype.insert = function*(collection, documents) {
  if (!_.isArray(documents)) {
    documents = [documents];
  }
  _.forEach(documents, function(document, i) {
    documents[i] = ensureKey(_.clone(document));
  });
  let c = this._db.collection(collection);
  yield query(this._db, c, aql `
    FOR document IN ${documents}
    INSERT document
    IN ${c}`);
}

/**
 * Find documents based on filter.
 * @param  {String} collection Collection name
 * @param  {Object} filter     Key, value Object
 * @return {array.Object}            A list of found documents.
 */
Arango.prototype.find = function*(collection, filter) {
  let c = this._db.collection(collection);
  if (_.size(filter) === 0) {
    return yield query(this._db, c, aql `
      FOR node in ${c}
        RETURN node
      `);
  }
  let filterStr = buildFilter(filter);
  let res = yield query(this._db, c, aql `
    FOR node in ${c}
      FILTER ${filterStr}
      RETURN node
    `);
  let docs = yield res.all();
  _.forEach(docs, function(doc) {
    sanitizeFields(doc);
  })
  return docs;
}

/**
 * Find documents by id (_key).
 * @param  {String} collection Collection name
 * @param  {String,array.String} ids        A single ID or multiple IDs.
 * @return {array.Object}            A list of found documents.
 */
Arango.prototype.findByID = function*(collection, ids) {
  if (!_.isArray(ids)) {
    ids = [ids];
  }
  let keys = new Array(ids.length);
  _.forEach(ids, function(id, i) {
    keys[i] = idToKey(id);
  });
  let c = this._db.collection(collection);<commit_msg>Sanitize each returning ArangoDB document<commit_after>'use strict';

var Arangojs = require('arangojs');
var aql = Arangojs.aql;
var util = require('util');
var slug = require('slug');
var _ = require('lodash');

const DB_SYSTEM = '_system';

/**
 * ArangoDB provider
 * @param {Object} conn Arangojs database connection.
 */
function Arango(conn) {
  this._db = conn;
}

function* query(db, c, q) {
  try {
    return yield db.query(q);
  } catch (err) {
    if (err.message && err.message.startsWith('collection not found')) {
      yield c.create();
    } else {
      throw err;
    }
  }
  return yield db.query(q);
}

function idToKey(id) {
  return slug(id, {
    replacement: '_'
  });
}

function ensureKey(document) {
  if (document._key) {
    return document;
  }
  let id = document.id;
  if (id) {
    document._key = idToKey(id);
  }
  return document;
}

function sanitizeFields(document) {
  delete document._id;
  delete document._key;
  delete document._rev;
  return document;
}

function buildFilter(filter, operator) {
  filter = ensureKey(filter);
  if (!operator) {
    operator = '&&';
  }
  let filterStr = '';
  _.forIn(filter, function(value, key) {
    switch (key) {
      case '$or':
        filterStr += buildFilter(value, '||');
      case '$and':
        filterStr += buildFilter(value, '&&');
      default:
        filterStr += util.format(' node.%s == %s %s', value, key, operator)
    }
  });
  filterStr = _.trimEnd(filterStr, operator);
  return filterStr;
}

/**
 * Inserts documents into database.
 * @param  {String} collection Collection name
 * @param  {Object|array.Object} documents  A single or multiple documents.
 */
Arango.prototype.insert = function*(collection, documents) {
  if (!_.isArray(documents)) {
    documents = [documents];
  }
  _.forEach(documents, function(document, i) {
    documents[i] = ensureKey(_.clone(document));
  });
  let c = this._db.collection(collection);
  yield query(this._db, c, aql `
    FOR document IN ${documents}
    INSERT document
    IN ${c}`);
}

/**
 * Find documents based on filter.
 * @param  {String} collection Collection name
 * @param  {Object} filter     Key, value Object
 * @return {array.Object}            A list of found documents.
 */
Arango.prototype.find = function*(collection, filter) {
  let c = this._db.collection(collection);
  if (_.size(filter) === 0) {
    let docs = yield query(this._db, c, aql `
      FOR node in ${c}
        RETURN node
      `);
    _.forEach(docs, function(doc) {
      sanitizeFields(doc);
    })
    return docs;
  }
  let filterStr = buildFilter(filter);
  let res = yield query(this._db, c, aql `
    FOR node in ${c}
      FILTER ${filterStr}
      RETURN node
    `);
  let docs = yield res.all();
  _.forEach(docs, function(doc) {
    sanitizeFields(doc);
  })
  return docs;
}

/**
 * Find documents by id (_key).
 * @param  {String} collection Collection name
 * @param  {String,array.String} ids        A single ID or multiple IDs.
 * @return {array.Object}            A list of found documents.
 */
Arango.prototype.findByID = function*(collection, ids) {
  if (!_.isArray(ids)) {
    ids = [ids];
  }
  let keys = new Array(ids.length);
  _.forEach(ids, function(id, i) {
    keys[i] = idToKey(id);
  });
  let c = this._db.collection(collection);
{'fraction_non_alphanumeric': 0.14960629921259844, 'fraction_numerical': 0.0003087849312953528, 'mean_word_length': 3.076777847702958, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1770861', 'n_tokens_mistral': 2132, 'n_tokens_neox': 2109, 'n_words': 567}