<commit_before>'use strict';

var _ = require('lodash');
var localforage = require('localforage');

// Provides class RequestCache, a wrapper around the localforage local storage provider
// that manages generating keys from XHR requests, and querying and saving data into the cache

var defaultOptions = {
  instanceName: null,
  keyPrefix: '',
  onStatus: null,
  onReady: null,
  debugCachePuts: true,
  debugCacheHits: true,
  debugCacheMiss: true,
  debugCacheBoot: true,
};

// Create a new RequestCache
// If options.instanceName is set and not null then use a new private scope of localforage of that name, otherwise use global localforage
// Using instanceName changes the behaviour of clear() to be less polite
function RequestCache(options) {
  console.log('[Teuthis] RequestCache constructor');
  var self = this;
  this.stats = {miss: 0, hits: 0, memory: 0};
  this.options = _.defaults({}, options);  _.defaults(this.options, defaultOptions);
  this.store = localforage;
  this.ownStore = false;
  if (this.options.instanceName !== null) {
    this.ownStore = true;
    this.store = localforage.createInstance({name: this.options.instanceName, description: 'Teuthis XHR proxy cache'});
  }

  // Keep our own list of keys, so we can flush them from localforage without
  // flushing other items that are not ours
  this.keyPrefix = (typeof this.options.keyprefix === 'string') ? this.options.keyprefix : '';
  this.cacheKeys = {};

  // If we have a key prefix or are using own store, then propulate cacheKeys
  // This is problematic, we cant guarantee that when the constructor returns, we are ready!
  this.ready = false;
  this.store.iterate(function(v, k) {
    if (self.ownStore || self.keyIsPrefixed(k)) {
      self.cacheKeys[k] = true;
      var m = 0;
      if (typeof v === 'string') m = v.length;
      else if (_.isArrayBuffer(v)) m = v.byteLength;
      // else console.log(v);
      self.stats.memory += m;
      if (self.options.debugCacheBoot) console.log('[Teuthis] found key: ' + k + ', memory: ' + m + '/' + self.stats.memory + ', ' + typeof v);
    }
  }, function() {
    console.log('[Teuthis] found keys: ' + Object.getOwnPropertyNames(self.cacheKeys).length);
    console.log('[Teuthis] found memory: ' + self.stats.memory);
    self.ready = true;
    if (self.options.onReady) self.options.onReady();
  });
}

function handleCacheMiss(key, onMiss) {
  if (this.options.debugCacheMiss) console.log('[Teuthis] proxy-miss ' + key);
  this.stats.miss ++;
  if (_.isFunction(onMiss)) onMiss(key);
  if (_.isFunction(this.options.onStatus)) this.options.onStatus.call(this);
}

function handleCacheHit(key, cachedValue, onHit) {
  if (this.options.debugCacheHits) console.log('[Teuthis] proxy-hit ' + key);
  this.stats.hits ++;
  if (_.isFunction(onHit)) onHit(key, cachedValue);
  if (_.isFunction(this.options.onStatus)) this.options.onStatus.call(this);
}

// Returns estimated number of entries in the cache
// If someone cleared the localforage instance in the meantime, number will be incorrect
RequestCache.prototype.weakLength = function() {
  return Object.getOwnPropertyNames(this.cacheKeys).length;
}

RequestCache.prototype.composeKey = function(method, url) {
  return this.keyPrefix + method + '__' + url;
}

RequestCache.prototype.keyIsPrefixed = function(key) {
  if (this.keyPrefix && this.keyPrefix.length > 0) {
    return key.startsWith(this.keyPrefix);
  } else {
    return false;
  }
}

// Reset statistical information (other than memory usage)
RequestCache.prototype.clearStats = function() {
  this.stats.miss = 0;
  this.stats.hits = 0;
}

RequestCache.prototype.getStats = function() {
  return Object.assign({}, this.stats);
}

// Clear all our entries from the localforage instance
// Probably slower than localforage.clear() but guarantee to remove only our entries
RequestCache.prototype.flush = function(done) {
  var self = this;
  if (this.ownStore) {
    self.store.clear(function() { self.cacheKeys = {}; self.stats.memory = 0; console.log('[Teuthis] proxy-flush'); if (done) done(); });
  } else {
    // Try and remove keys we know about, or with our prefix
    self.store.iterate(function(v, k) {
      if (self.cacheKeys.hasOwnProperty(k) || self.keyIsPrefixed(k)) {
        self.store.remove(k, function() { delete self.cacheKeys[k]; });
      }
    }, function() {
      self.cacheKeys = {};
      self.stats.memory = 0;
      console.log('[Teuthis] proxy-flush');
      if (done) { done(); }
    });
  }
}

RequestCache.prototype.each = function(cb, done) {
  var self = this;
  self.store.iterate(function(v, k) {
    if (self.cacheKeys.hasOwnProperty(k) || self.keyIsPrefixed(k)) {
      cb(k, v);
    }
  }, done);
}

// Weakly check if object is cached. Only checks our key list, not localforage itself
// So does not guarantee match(...) will have a HIT if, say, someone cleared localforage
RequestCache.prototype.weakHas = function(method, url) {
  var key = this.composeKey(method, url);
  return this.cacheKeys.hasOwnProperty(key);
}

// If method:url is in localforage then call onHit(composedKey, response) else call onMiss(composedKey)
RequestCache.prototype.match = function(method, url, onHit, onMiss) {
  var key = this.composeKey(method, url);
  var self = this;
  this.store.getItem(key)
    .then(function(cachedValue) {
      if (cachedValue === null) {
        delete self.cacheKeys[key]; // Handle the case where something else managed to delete an entry from localforage
        handleCacheMiss.call(self, key, onMiss);
      } else {
        handleCacheHit.call(self, key, cachedValue, onHit);
      }
    }).catch(function(err) {
      console.error('[Teuthis] proxy-cache-match error ' + err);
      console.error(err);
      delete self.cacheKeys[key];
      handleCacheMiss.call(self, key, onMiss);
    });
}

// Put value of response for method:url into localforage, and call done(), or call done(err) if an error happens
RequestCache.prototype.put = function(method, url, value, done) {
  var key = this.composeKey(method, url);
  if (this.options.debugCachePuts) console.log('[Teuthis] proxy-cache-put ' + key);
  var self = this;
  this.store.setItem(key, value).then(function() {
    self.cacheKeys[key] = true;
    if (typeof value === 'string') self.stats.memory += value.length;
    else if (_.isArrayBuffer(value)) self.stats.memory += value.byteLength;
    if (done) done();
  }).catch(function(err) {
    console.error('[Teuthis] proxy-cache-put error ' + err);
    if (done) done(err);
  });
}

RequestCache.prototype.forceClear = function() { this.store.clear(); }

module.exports = RequestCache;<commit_msg>Catch some errors and improve handling<commit_after>'use strict';

var _ = require('lodash');
var localforage = require('localforage');

// Provides class RequestCache, a wrapper around the localforage local storage provider
// that manages generating keys from XHR requests, and querying and saving data into the cache

var defaultOptions = {
  instanceName: null,
  keyPrefix: '',
  onStatus: null,
  onReady: null,
  debugCachePuts: true,
  debugCacheHits: true,
  debugCacheMiss: true,
  debugCacheBoot: true,
};

// Create a new RequestCache
// If options.instanceName is set and not null then use a new private scope of localforage of that name, otherwise use global localforage
// Using instanceName changes the behaviour of clear() to be less polite
function RequestCache(options) {
  console.log('[Teuthis] RequestCache constructor');
  var self = this;
  this.stats = {miss: 0, hits: 0, memory: 0};
  this.options = _.defaults({}, options);  _.defaults(this.options, defaultOptions);
  this.store = localforage;
  this.ownStore = false;
  if (this.options.instanceName !== null) {
    this.ownStore = true;
    this.store = localforage.createInstance({name: this.options.instanceName, description: 'Teuthis XHR proxy cache'});
  }

  // Keep our own list of keys, so we can flush them from localforage without
  // flushing other items that are not ours
  this.keyPrefix = (typeof this.options.keyprefix === 'string') ? this.options.keyprefix : '';
  this.cacheKeys = {};

  // If we have a key prefix or are using own store, then propulate cacheKeys
  // This is problematic, we cant guarantee that when the constructor returns, we are ready!
  this.ready = false;
  this.store.iterate(function(v, k) {
    if (self.ownStore || self.keyIsPrefixed(k)) {
      self.cacheKeys[k] = true;
      var m = 0;
      if (typeof v === 'string') m = v.length;
      else if (_.isArrayBuffer(v)) m = v.byteLength;
      // else console.log(v);
      self.stats.memory += m;
      if (self.options.debugCacheBoot) console.log('[Teuthis] found key: ' + k + ', memory: ' + m + '/' + self.stats.memory + ', ' + typeof v);
    }
  }, function() {
    console.log('[Teuthis] found keys: ' + Object.getOwnPropertyNames(self.cacheKeys).length);
    console.log('[Teuthis] found memory: ' + self.stats.memory);
    self.ready = true;
    if (self.options.onReady) self.options.onReady();
  });
}

function handleCacheMiss(key, onMiss) {
  if (this.options.debugCacheMiss) console.log('[Teuthis] proxy-miss ' + key);
  this.stats.miss ++;
  if (_.isFunction(onMiss)) onMiss(key);
  if (_.isFunction(this.options.onStatus)) this.options.onStatus.call(this);
}

function handleCacheHit(key, cachedValue, onHit) {
  if (this.options.debugCacheHits) console.log('[Teuthis] proxy-hit ' + key);
  this.stats.hits ++;
  if (_.isFunction(onHit)) onHit(key, cachedValue);
  if (_.isFunction(this.options.onStatus)) this.options.onStatus.call(this);
}

// Returns estimated number of entries in the cache
// If someone cleared the localforage instance in the meantime, number will be incorrect
RequestCache.prototype.weakLength = function() {
  return Object.getOwnPropertyNames(this.cacheKeys).length;
}

RequestCache.prototype.composeKey = function(method, url) {
  return this.keyPrefix + method + '__' + url;
}

RequestCache.prototype.keyIsPrefixed = function(key) {
  if (this.keyPrefix && this.keyPrefix.length > 0) {
    return key.startsWith(this.keyPrefix);
  } else {
    return false;
  }
}

// Reset statistical information (other than memory usage)
RequestCache.prototype.clearStats = function() {
  this.stats.miss = 0;
  this.stats.hits = 0;
}

RequestCache.prototype.getStats = function() {
  return Object.assign({}, this.stats);
}

// Clear all our entries from the localforage instance
// Probably slower than localforage.clear() but guarantee to remove only our entries
RequestCache.prototype.flush = function(done) {
  var self = this;
  if (this.ownStore) {
    self.store.clear(function() { self.cacheKeys = {}; self.stats.memory = 0; console.log('[Teuthis] proxy-flush'); if (done) done(); });
  } else {
    // Try and remove keys we know about, or with our prefix
    self.store.iterate(function(v, k) {
      if (self.cacheKeys.hasOwnProperty(k) || self.keyIsPrefixed(k)) {
        self.store.remove(k, function() { delete self.cacheKeys[k]; });
      }
    }, function() {
      self.cacheKeys = {};
      self.stats.memory = 0;
      console.log('[Teuthis] proxy-flush');
      if (done) { done(); }
    });
  }
}

RequestCache.prototype.each = function(cb, done) {
  var self = this;
  self.store.iterate(function(v, k) {
    if (self.cacheKeys.hasOwnProperty(k) || self.keyIsPrefixed(k)) {
      cb(k, v);
    }
  }, done);
}

// Weakly check if object is cached. Only checks our key list, not localforage itself
// So does not guarantee match(...) will have a HIT if, say, someone cleared localforage
RequestCache.prototype.weakHas = function(method, url) {
  var key = this.composeKey(method, url);
  return this.cacheKeys.hasOwnProperty(key);
}

// If method:url is in localforage then call onHit(composedKey, response) else call onMiss(composedKey)
RequestCache.prototype.match = function(method, url, onHit, onMiss) {
  var key = this.composeKey(method, url);
  var self = this;
  this.store.getItem(key)
    .then(function(cachedValue) {
      try {
        if (cachedValue === null) {
          delete self.cacheKeys[key]; // Handle the case where something else managed to delete an entry from localforage
          handleCacheMiss.call(self, key, onMiss);
        } else {
          handleCacheHit.call(self, key, cachedValue, onHit);
        }
      } catch (err) {
        // callback was source of the error, not this.store.getItem
        console.error('[Teuthis] proxy-cache-match handler error ' + err);
        console.error(err);
        delete self.cacheKeys[key];
      }
    }).catch(function(err) {
      // care - this will catch errors in the handler, not just errors in the cache itself
      //      - if we are not careful by having try/catch above
      //      - i.e. we need to avoid a double call to handleCacheMiss
      // otherwise we see -
      //    Error: Uncaught (in promise): InvalidStateError: XMLHttpRequest state must be OPENED.
      //    Derived from xhr.send.apply(xhr, arguments); in send() inside the miss callback
      console.error('[Teuthis] proxy-cache-match error ' + err);
      console.error(err);
      delete self.cacheKeys[key];
      handleCacheMiss.call(self, key, onMiss);
    });
}

// Put value of response for method:url into localforage, and call done(), or call done(err) if an error happens
RequestCache.prototype.put = function(method, url, value, done) {
  var key = this.composeKey(method, url);
  if (this.options.debugCachePuts) console.log('[Teuthis] proxy-cache-put ' + key);
  var self = this;
  this.store.setItem(key, value).then(function() {
    self.cacheKeys[key] = true;
    if (typeof value === 'string') self.stats.memory += value.length;
    else if (_.isArrayBuffer(value)) self.stats.memory += value.byteLength;
    if (done) done();
  }).catch(function(err) {
    console.error('[Teuthis] proxy-cache-put error ' + err);
    if (done) done(err);
  });
}

RequestCache.prototype.forceClear = function() { this.store.clear(); }

module.exports = RequestCache;
{'fraction_non_alphanumeric': 0.13374338435130884, 'fraction_numerical': 0.0012873694750393362, 'mean_word_length': 3.620951751487112, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '878212', 'n_tokens_mistral': 4352, 'n_tokens_neox': 4327, 'n_words': 1218}