<commit_before>
#include <sstream>
#include <algorithm>

#include "src/database/database.h"

#include "mongo/client/dbclient.h"
#include "mongo/bson/bson.h"

#include "include/container/tokenizer.h"

using namespace std;
using namespace stoke;
using namespace x64asm;
using namespace mongo;
using namespace cpputil;

void Database::insert(x64asm::Code code, std::string tag) {

  stringstream code_ss;
  code_ss << code;

  string db = database_ + "." + tag;

  try {
    /* Either add this code to the database with
       count 1, or update the existing count */
    connection_.ensureIndex(db, fromjson("{code:1}"));
    connection_.update(db,
             BSON( "code" << code_ss.str() ),
             BSON( "$inc" << BSON( "count" << 1 ) ),
             true);

  } catch (const DBException &e) {
    cerr << "caught " << e.what() << endl;
  }

}

uint64_t Database::lookup(x64asm::Code code, string tag) {

  stringstream code_ss;
  code_ss << code;

  string db = database_ + "." + tag;

  try {
    /* Lookup the count corresponding to this
       code/tag pair */
    DBClientCursor *cursor = 
      connection_.query(db, QUERY("code" << code_ss.str())).release();

    uint64_t sum = 0;
    while (cursor->more()) {
      BSONObj p = cursor->next();
      sum += p.getIntField("count");
    }
    
    return sum;

  } catch (const DBException &e) {
    cerr << "caught " << e.what() << endl;
    return 0;
  }

}

Database::Database(string hostname, uint16_t port, string database) {

  //TODO use the port :)
  connection_.connect(hostname);
  database_ = database;

  // Wipe out anything that's ready there 
  connection_.eval(database, "db.dropDatabase();");

}<commit_msg>fixed the huge memory leak<commit_after>
#include <sstream>
#include <algorithm>

#include "src/database/database.h"

#include "mongo/client/dbclient.h"
#include "mongo/bson/bson.h"

#include "include/container/tokenizer.h"

using namespace std;
using namespace stoke;
using namespace x64asm;
using namespace mongo;
using namespace cpputil;

void Database::insert(x64asm::Code code, std::string tag) {

  stringstream code_ss;
  code_ss << code;

  stringstream db_ss;
  db_ss << database_ << "." << tag;
  string db = db_ss.str();

  try {
    /* Either add this code to the database with
       count 1, or update the existing count */
    connection_.ensureIndex(db, fromjson("{code:1}"));
    connection_.update(db,
             BSON( "code" << code_ss.str() ),
             BSON( "$inc" << BSON( "count" << 1 ) ),
             true);

  } catch (const DBException &e) {
    cerr << "caught " << e.what() << endl;
  }

}

uint64_t Database::lookup(x64asm::Code code, string tag) {

  stringstream code_ss;
  code_ss << code;

  stringstream db_ss;
  db_ss << database_ << "." << tag;
  string db = db_ss.str();

  try {
    /* Lookup the count corresponding to this
       code/tag pair */
    DBClientCursor *cursor = 
      connection_.query(db, QUERY("code" << code_ss.str())).release();

    uint64_t sum = 0;
    while (cursor->more()) {
      BSONObj p = cursor->next();
      sum += p.getIntField("count");
    }

    delete cursor;
    
    return sum;

  } catch (const DBException &e) {
    cerr << "caught " << e.what() << endl;
    return 0;
  }

}

Database::Database(string hostname, uint16_t port, string database) {

  //TODO use the port :)
  connection_.connect(hostname);
  database_ = database;

  // Wipe out anything that's ready there 
  connection_.eval(database, "db.dropDatabase();");

}
{'fraction_non_alphanumeric': 0.14579172610556348, 'fraction_numerical': 0.009700427960057061, 'mean_word_length': 3.0578703703703702, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3186499', 'n_tokens_mistral': 1203, 'n_tokens_neox': 1150, 'n_words': 308}