<commit_before>/* jshint camelcase: false */

"use strict";

var mongo  = require('mongodb'),
    assert = require('assert'),
    _      = require('underscore'),
    unorm  = require('unorm'),
    regexp_quote      = require('regexp-quote'),
    DoubleMetaphone = require('doublemetaphone'),
    dm = new DoubleMetaphone();

var server      = new mongo.Server('localhost', 27017, {auto_reconnect: true});
var mongoclient = new mongo.MongoClient(server, {journal: true});

mongoclient.open(function (err) {
  if (err) {
    throw err;
  }
});

function Storage(databaseName) {
  assert(databaseName, "Need to provide a database name");
  this.databaseName = databaseName;
  this.db = mongoclient.db(databaseName);
  this.fields = {all: {}};
}

Storage.generateID = function () {
  var objectId = new mongo.ObjectID();
  return objectId.toHexString();
};

function deduplicate_slug(collection, cb) {
  /*jshint validthis:true */
  var self = this;

  if (!self.slug) {
    return cb(null);
  }

  // find other entries in the database that have the same slug
  collection.findOne({ slug: self.slug, _id: { $ne: self.id } }, function(err, doc) {
    if (err) {
      return cb(err);
    }

    // if nothing found then no need to change slug
    if ( ! doc ) {
      return cb(null);
    }

    // we have a conflict, increment the slug
    var matches = self.slug.match(/^(.*)\-(\d+)$/);

    if ( !matches ) {
      self.slug = self.slug + '-1';
    } else {
      var base_slug = matches[1];
      var counter   = parseInt( matches[2], 10 ) + 1;
      self.slug     = base_slug + '-' + counter;
    }

    return deduplicate_slug.apply( self, [ collection, cb ] ); // recurse

  });
}

function indexNameWords(doc, v) {
  // Set normalized array of words for searching
  var words = _.union(
    v.split(/\s+/).map( function(s) {
      return s.toLowerCase();
    }),
    v.split(/\s+/).map( function(s) {
      return unorm.nfkd(s.toLowerCase()).replace(/[\u0300-\u036F]/g, '');
    })
  );
  doc._internal = doc._internal || {};
  doc._internal.name_words = words;
  // Set the double metaphone entries. Currently just stores both primary and secondary without saying which is which
  var dm_words = [];
  words.forEach(function(w) {
    dm_words.push.apply(dm_words, _.values(dm.doubleMetaphone(w)) );
  });
  // Also include the words to make the searching easier. Perhaps this can be just one array?
  doc._internal.name_dm = dm_words.concat(words);
}

/*
  Store a document in the database.
*/
Storage.prototype.store = function (collectionName, doc, cb) {

  var fields = this.fields;

  if (!doc.id) {
    return cb(new Error("Can't store document without an id"));
  }

  if (collectionName === 'persons') {
    indexNameWords(doc, doc.name);
  }

  var collection = this.db.collection(collectionName);

  deduplicate_slug.apply(doc, [ collection, function() {
    var docToStore = _.extend({}, doc, {_id: doc.id});
    collection.update({_id: doc.id}, docToStore, {upsert: true}, function (err, result) {
      assert(result);
      cb(err, filterDoc(doc, fields));
    });
  } ] );

};

/*
  Retrieve a document from the database.<commit_msg>Index all collections that have a name field<commit_after>/* jshint camelcase: false */

"use strict";

var mongo  = require('mongodb'),
    assert = require('assert'),
    _      = require('underscore'),
    unorm  = require('unorm'),
    regexp_quote      = require('regexp-quote'),
    DoubleMetaphone = require('doublemetaphone'),
    dm = new DoubleMetaphone();

var server      = new mongo.Server('localhost', 27017, {auto_reconnect: true});
var mongoclient = new mongo.MongoClient(server, {journal: true});

mongoclient.open(function (err) {
  if (err) {
    throw err;
  }
});

function Storage(databaseName) {
  assert(databaseName, "Need to provide a database name");
  this.databaseName = databaseName;
  this.db = mongoclient.db(databaseName);
  this.fields = {all: {}};
}

Storage.generateID = function () {
  var objectId = new mongo.ObjectID();
  return objectId.toHexString();
};

function deduplicate_slug(collection, cb) {
  /*jshint validthis:true */
  var self = this;

  if (!self.slug) {
    return cb(null);
  }

  // find other entries in the database that have the same slug
  collection.findOne({ slug: self.slug, _id: { $ne: self.id } }, function(err, doc) {
    if (err) {
      return cb(err);
    }

    // if nothing found then no need to change slug
    if ( ! doc ) {
      return cb(null);
    }

    // we have a conflict, increment the slug
    var matches = self.slug.match(/^(.*)\-(\d+)$/);

    if ( !matches ) {
      self.slug = self.slug + '-1';
    } else {
      var base_slug = matches[1];
      var counter   = parseInt( matches[2], 10 ) + 1;
      self.slug     = base_slug + '-' + counter;
    }

    return deduplicate_slug.apply( self, [ collection, cb ] ); // recurse

  });
}

function indexNameWords(doc, v) {
  // Set normalized array of words for searching
  var words = _.union(
    v.split(/\s+/).map( function(s) {
      return s.toLowerCase();
    }),
    v.split(/\s+/).map( function(s) {
      return unorm.nfkd(s.toLowerCase()).replace(/[\u0300-\u036F]/g, '');
    })
  );
  doc._internal = doc._internal || {};
  doc._internal.name_words = words;
  // Set the double metaphone entries. Currently just stores both primary and secondary without saying which is which
  var dm_words = [];
  words.forEach(function(w) {
    dm_words.push.apply(dm_words, _.values(dm.doubleMetaphone(w)) );
  });
  // Also include the words to make the searching easier. Perhaps this can be just one array?
  doc._internal.name_dm = dm_words.concat(words);
}

/*
  Store a document in the database.
*/
Storage.prototype.store = function (collectionName, doc, cb) {

  var fields = this.fields;

  if (!doc.id) {
    return cb(new Error("Can't store document without an id"));
  }

  if (doc.name) {
    indexNameWords(doc, doc.name);
  }

  var collection = this.db.collection(collectionName);

  deduplicate_slug.apply(doc, [ collection, function() {
    var docToStore = _.extend({}, doc, {_id: doc.id});
    collection.update({_id: doc.id}, docToStore, {upsert: true}, function (err, result) {
      assert(result);
      cb(err, filterDoc(doc, fields));
    });
  } ] );

};

/*
  Retrieve a document from the database.
{'fraction_non_alphanumeric': 0.15197087224948552, 'fraction_numerical': 0.005698907709355707, 'mean_word_length': 3.078760490639122, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7007750', 'n_tokens_mistral': 2128, 'n_tokens_neox': 2071, 'n_words': 543}