<commit_before>var fs = require('fs');

exports.insert = function( type, key, data, layerId, callback ){
  Cache.db.insert( type+':'+key, data, layerId, function(err, success){
    Cache.db.timer.set( type+':'+key+':timer', 3600000, function( error, timer){
      callback( err, success );
    });
  });
};

exports.insertPartial = function( type, key, data, layerId, callback ){
  Cache.db.insertPartial( type+':'+key, data, layerId, function(err, success){
    callback( err, success );
  });
};


// handles the response from the select
// if a cache timer has expired we ask if the data in the 3rd party API has changed 
// if it has changed then each Model's checkCache function will return the NEW data 
// if it hasnt changed then checkCache will return false and the old data will be sent back   
exports.process = function( type, key, data, options, callback ){
  var self = this;
  // timing at which we'll check the validity of the cache 
  var checkTime = (60*60*1000); // 60 mins 

  if ( !data.length ){
    callback( 'Not found', null);
  } else {
    // checks the timer 
    var timerKey = [type, key, (options.layer || 0), 'timer'].join(':');
    Cache.db.timer.get( timerKey, function( error, timer){
      if ( timer ){
        // got a timer, therefore we are good and just return
        callback( null, data );
      } else {
        // expired, hit the API to check the latest sha
        if ( global[type] && global[type].checkCache ) {
          global[type].checkCache( key, data, options, function(err, success){
            if ( !success ){
              // false is good -> reset timer and return data
              // cache returned true, return current data
              Cache.db.timer.set( timerKey, checkTime, function( error, timer){
                callback( null, data );
              });
            } else {
              // we need to remove and save new data 
              self.remove(type, key, options, function(){
                self.insert(type, key, data, (options.layer || 0), function(err, res){
                  Cache.db.timer.set( timerKey, checkTime, function( error, timer){
                    callback( err, success );
                  });
                });
              });
            } 
          });
        } else {
          callback( null, data ); 
        }
      }
    });
  }
};

exports.remove = function(type, key, options, callback){
  var self = this;
  Cache.db.remove( type+':'+key+':'+(options.layer || 0), function(err, result){
    // Remove all files
    self.rmDir( [ config.data_dir+'files/'+type, key, (options.layer || 0) ].join(':'), function(err, res){
      self.rmDir( [ config.data_dir+'tiles/'+type, key, (options.layer || 0) ].join(':'), function(err, res){
        self.rmDir( [ config.data_dir+'thumbs/'+type, key, (options.layer || 0) ].join(':'), function(err, res){
          if (config.peechee){
            console.log('must rm bucket subdir on s3');
          }
          if ( callback ) { 
            callback(null, true);
          }
        });
      });
    });
  });
};

exports.rmDir = function(dir, callback){
  fs.rmdir( dir, callback);
};

exports.get = function(type, key, options, callback ){
  var self = this;
  Cache.db.select( type+':'+key, options, function(err, result){
    self.process( type, key, result, options, callback );     
  });
};

exports.getInfo = function( key, callback ){
  Cache.db.getInfo( key, callback );
};

exports.updateInfo = function( key, info, callback ){
  Cache.db.updateInfo( key, info, callback );
};

<commit_msg>rm console<commit_after>var fs = require('fs');

exports.insert = function( type, key, data, layerId, callback ){
  Cache.db.insert( type+':'+key, data, layerId, function(err, success){
    Cache.db.timer.set( type+':'+key+':timer', 3600000, function( error, timer){
      callback( err, success );
    });
  });
};

exports.insertPartial = function( type, key, data, layerId, callback ){
  Cache.db.insertPartial( type+':'+key, data, layerId, function(err, success){
    callback( err, success );
  });
};


// handles the response from the select
// if a cache timer has expired we ask if the data in the 3rd party API has changed 
// if it has changed then each Model's checkCache function will return the NEW data 
// if it hasnt changed then checkCache will return false and the old data will be sent back   
exports.process = function( type, key, data, options, callback ){
  var self = this;
  // timing at which we'll check the validity of the cache 
  var checkTime = (60*60*1000); // 60 mins 

  if ( !data.length ){
    callback( 'Not found', null);
  } else {
    // checks the timer 
    var timerKey = [type, key, (options.layer || 0), 'timer'].join(':');
    Cache.db.timer.get( timerKey, function( error, timer){
      if ( timer ){
        // got a timer, therefore we are good and just return
        callback( null, data );
      } else {
        // expired, hit the API to check the latest sha
        if ( global[type] && global[type].checkCache ) {
          global[type].checkCache( key, data, options, function(err, success){
            if ( !success ){
              // false is good -> reset timer and return data
              // cache returned true, return current data
              Cache.db.timer.set( timerKey, checkTime, function( error, timer){
                callback( null, data );
              });
            } else {
              // we need to remove and save new data 
              self.remove(type, key, options, function(){
                self.insert(type, key, data, (options.layer || 0), function(err, res){
                  Cache.db.timer.set( timerKey, checkTime, function( error, timer){
                    callback( err, success );
                  });
                });
              });
            } 
          });
        } else {
          callback( null, data ); 
        }
      }
    });
  }
};

exports.remove = function(type, key, options, callback){
  var self = this;
  Cache.db.remove( type+':'+key+':'+(options.layer || 0), function(err, result){
    // Remove all files
    self.rmDir( [ config.data_dir+'files/'+type, key, (options.layer || 0) ].join(':'), function(err, res){
      self.rmDir( [ config.data_dir+'tiles/'+type, key, (options.layer || 0) ].join(':'), function(err, res){
        self.rmDir( [ config.data_dir+'thumbs/'+type, key, (options.layer || 0) ].join(':'), function(err, res){
          if ( callback ) { 
            callback(null, true);
          }
        });
      });
    });
  });
};

exports.rmDir = function(dir, callback){
  fs.rmdir( dir, callback);
};

exports.get = function(type, key, options, callback ){
  var self = this;
  Cache.db.select( type+':'+key, options, function(err, result){
    self.process( type, key, result, options, callback );     
  });
};

exports.getInfo = function( key, callback ){
  Cache.db.getInfo( key, callback );
};

exports.updateInfo = function( key, info, callback ){
  Cache.db.updateInfo( key, info, callback );
};


{'fraction_non_alphanumeric': 0.15092460881934566, 'fraction_numerical': 0.006970128022759601, 'mean_word_length': 2.375420067210754, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3981201', 'n_tokens_mistral': 2117, 'n_tokens_neox': 2036, 'n_words': 649}