<commit_before>const mongoose = require('mongoose');
const _ = require('lodash-fp');
const asap = require('asap');
const sortedObject = require('sorted-object');

const schema = new mongoose.Schema({
  _ranking: {
    type: Number,
    required: true,
    index: true
  },
  _source: {
    type: String,
    required: true,
    index: true
  },
  _origin: {
    type: String,
    required: true
  },
  _host: {
    type: String,
    required: true,
    index: true,
  },
  url: {
    type: String,
    required: true,
  },
  title: {
    type: String,
    required: true
  },
  image: {
    type: String
  },
  createdAt: {
    type: Date,
    default: Date.now,
    expires: 86400, // ttl in seconds
    required: true
  }
});

schema.set('toObject', {
  transform(doc, ret, options) {
    // delete ret._id;
    delete ret.__v;
    // delete ret.createdAt; enable this if you do not want to update the expiry
  }
});

schema.set('toJSON', {
  transform(doc, ret, options) {
    // delete ret._id;
    delete ret.__v;
    // delete ret.createdAt; enable this if you do not want to update the expiry
  }
});

schema.statics.compareToPrevious = function(entry, cb) {
  if (!entry._source) return cb();

  this.findOne({ _source: entry._source })
    .sort({ createdAt: -1 })
    .exec(function(err, result) {
      if (err) return cb(err);

      let normalizedResult = (result || {}).toJSON ? result.toJSON() : {};
      let formatObject = _.pick(['title', 'image', 'url']);
      let existingEntry = JSON.stringify(sortedObject(formatObject(normalizedResult)));
      let newEntry = JSON.stringify(sortedObject(formatObject(entry)));

      cb(null, existingEntry !== newEntry);
    });
};

// create the model for articles and return it
module.exports = mongoose.model('entry', schema);<commit_msg>NOISSUE: Add method for finding the latest entry of a query<commit_after>const mongoose = require('mongoose');
const _ = require('lodash-fp');
const asap = require('asap');
const sortedObject = require('sorted-object');

const schema = new mongoose.Schema({
  _ranking: {
    type: Number,
    required: true,
    index: true
  },
  _source: {
    type: String,
    required: true,
    index: true
  },
  _origin: {
    type: String,
    required: true
  },
  _host: {
    type: String,
    required: true,
    index: true,
  },
  url: {
    type: String,
    required: true,
  },
  title: {
    type: String,
    required: true
  },
  image: {
    type: String
  },
  createdAt: {
    type: Date,
    default: Date.now,
    expires: 86400, // ttl in seconds
    required: true
  }
});

schema.set('toObject', {
  transform(doc, ret, options) {
    // delete ret._id;
    delete ret.__v;
    // delete ret.createdAt; enable this if you do not want to update the expiry
  }
});

schema.set('toJSON', {
  transform(doc, ret, options) {
    // delete ret._id;
    delete ret.__v;
    // delete ret.createdAt; enable this if you do not want to update the expiry
  }
});

schema.statics.compareToPrevious = function(entry, cb) {
  if (!entry._source) return cb();

  this.findOne({ _source: entry._source })
    .sort({ createdAt: -1 })
    .exec((err, result) => {
      if (err) return cb(err);

      let normalizedResult = (result || {}).toJSON ? result.toJSON() : {};
      let formatObject = _.pick(['title', 'image', 'url']);
      let existingEntry = JSON.stringify(sortedObject(formatObject(normalizedResult)));
      let newEntry = JSON.stringify(sortedObject(formatObject(entry)));

      cb(null, existingEntry !== newEntry);
    });
};

schema.statics.findLatest = function(query, cb) {
  if (!query) return cb();

  this.findOne(query)
    .sort({ createdAt: -1 })
    .exec(cb);
};

// create the model for articles and return it
module.exports = mongoose.model('entry', schema);
{'fraction_non_alphanumeric': 0.148324096067564, 'fraction_numerical': 0.003430984428609132, 'mean_word_length': 2.9727463312368974, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1405125', 'n_tokens_mistral': 1245, 'n_tokens_neox': 1230, 'n_words': 335}