<commit_before>var bake = require('bake');
var Reg = require('reg');
var pandoc = require('pdc');
var isodate = require('isodate');
var mongo = require('mongodb');
var append = require('append');
var clone = require('clone');
var Set = require('Set');

var write = require('./write.js');

// empty callback
var x = function () {};

// Main function
var bread = function(conf, cb) {

  var dir = conf.directories;

  // Open connection to MongoDB
  new mongo.Db(conf.db.name, new mongo.Server(conf.db.host, conf.db.port))
      .open(function (err, db) {
    if (err)
      return cb(err);

    // Open collection
    db.collection(conf.db.collection, function (err, collection) {
      if (err)
        return cb(err);

      // Get registry
      new Reg(collection, function (err, reg) {
        if (err)
          return cb(err);

        var hooks = {};
        reg.tags = new Set();

        // Parse contents with `marked()`
        hooks.__content = function __content(f, prop, cb) {
          pandoc(prop.__content, 'markdown', 'html', cb);
        };

        hooks.__propBefore = function __propBefore(f, prop, cb) {
          if (!prop.modified)
            prop.modified = new Date();

          cb(null, prop);
        };

        // Parse ISO 8601 dates
        hooks.created = function(f, prop, cb) {
          var created;
          // If it's not a date, try parse it
          if (typeof prop.created != 'date') {
            try {
              created = isodate(prop.created);
            } catch (err) {
              try {
                created = new Date(prop.created);
              } catch (err) {
                return cb(err);
              }
            }
          } else
            created = prop.created;

          cb(null, created);
        };

        // Save files in registry
        hooks.__writeAfter = function(f, prop, cb) {
          var id = prop._id;
          delete prop._id;
          reg.extend(id, prop, x);
          if (prop.tags)
            for (var i = 0; i < prop.tags.length; i++)
              reg.tags.add(prop.tags[i].trim());
          cb(null, prop);<commit_msg>Handle dates properly<commit_after>var bake = require('bake');
var Reg = require('reg');
var pandoc = require('pdc');
var isodate = require('isodate');
var mongo = require('mongodb');
var append = require('append');
var clone = require('clone');
var Set = require('Set');

var write = require('./write.js');

// empty callback
var x = function () {};

// Main function
var bread = function(conf, cb) {

  var dir = conf.directories;

  // Open connection to MongoDB
  new mongo.Db(conf.db.name, new mongo.Server(conf.db.host, conf.db.port))
      .open(function (err, db) {
    if (err)
      return cb(err);

    // Open collection
    db.collection(conf.db.collection, function (err, collection) {
      if (err)
        return cb(err);

      // Get registry
      new Reg(collection, function (err, reg) {
        if (err)
          return cb(err);

        var hooks = {};
        reg.tags = new Set();

        // Parse contents with `marked()`
        hooks.__content = function __content(f, prop, cb) {
          pandoc(prop.__content, 'markdown', 'html', cb);
        };

        hooks.__propBefore = function __propBefore(f, prop, cb) {
          if (!prop.modified)
            prop.modified = new Date();

          cb(null, prop);
        };

        function fixDate(date, cb) {
          var created;
          // If it's not a date, try to parse it
          if (typeof date != 'date' && typeof date != 'undefined') {
            try {
              date = isodate(date);
            } catch (err) {
              try {
                date = new Date(date);
              } catch (err) {
                return cb(err);
              }
            }
          }

          cb(null, date);
        }

        hooks.created = function created(f, prop, cb) {
          fixDate(prop.created, cb);
        };

        hooks.modified = function modified(f, prop, cb) {
          fixDate(prop.modified, cb);
        };

        hooks.__writeAfter = function __writeAfter(f, prop, cb) {
          var id = prop._id;
          delete prop._id;
          reg.extend(id, prop, x);
          if (prop.tags)
            for (var i = 0; i < prop.tags.length; i++)
              reg.tags.add(prop.tags[i].trim());
          cb(null, prop);
{'fraction_non_alphanumeric': 0.1401419738951225, 'fraction_numerical': 0.0013739409205404168, 'mean_word_length': 1.745443117536141, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1125372', 'n_tokens_mistral': 1307, 'n_tokens_neox': 1270, 'n_words': 360}