<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */


var 
logger = require('winston'),
sha1 = require('sha1');

require('../../lib/extensions/number');


// TODO:  "value", This should be backed by a linked-list in redis, mysql or some 
//		    method that doesn't just exist in memory. A bloom filter could be 
//        nice.

// TODO: This could use some tests

module.exports = function(){
  this.keys = {};

  this.hasntHeardOf = function(key){
    return (!(key in this.keys));
  }
  this.add = function(key){
    if (!(key in this.keys))
      this.keys[key] = new Date();

    return;
  }
  this.prune = function(){
    logger.debug('Pruning!');
    for (var key in this.keys){
      //TODO: Go back to 1 hour
      if (new Date() - this.keys[key] > (30).hour()){<commit_msg>Using a hash instead of full IDs. This *might* be more memory-efficient, it's certainly more predictable.<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this file,
 * You can obtain one at http://mozilla.org/MPL/2.0/. */


var 
logger = require('winston'),
crypto = require('crypto');



require('../../lib/extensions/number');


// TODO:  "value", This should be backed by a linked-list in redis, mysql or some 
//		    method that isn't volatile  doesn't just exist in memory. A bloom filter could be 
//        nice.

// TODO: This could use some tests

module.exports = function(ageWindow){
  this.keys = {};

  this.hasntHeardOf = function(key){
    return (!(key in this.keys));
  }
  this.add = function(key){
    key = crypto.createHash('md5').update(key).digest();
    if (!(key in this.keys))
      this.keys[key] = new Date();

    return;
  }
  this.prune = function(){
    logger.debug('Pruning!');
    for (var key in this.keys){
      //TODO: Go back to 1 hour
      if (new Date() - this.keys[key] > ageWindow){
{'fraction_non_alphanumeric': 0.14505283381364073, 'fraction_numerical': 0.007204610951008645, 'mean_word_length': 2.8360957642725597, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '469699', 'n_tokens_mistral': 682, 'n_tokens_neox': 644, 'n_words': 242}