<commit_before>// Main Cache Interface
const Promise = require('bluebird');
const hash = require('object-hash');
const moment = require('moment');
const _ = require('lodash');

const Value = require('./value');
const NotFoundError = require('./errors/notFound');

const defaultOptions = {
  name: 'default',
  bypassHeader: 'cache-bypass',
  statusHeader: 'cache-status',
  includeMethods: ['GET']
};

/**
 * A cache object
 *
 * Takes in an array of stores and a hash of options
 **/
function Cache(stores = [], options = {}) {
  if(!(stores instanceof Array)) {
    stores = [stores];
  }

  this.stores = stores;
  this.options = Object.assign({},
                               defaultOptions,
                               options);
};

// Returns the list of stores
Cache.prototype.getStores = function getStores() {
  return this.stores;
};

// A key generator based on [object-hash](https://github.com/puleos/object-hash)
Cache.prototype.getKey = function getKey(obj = {}) {
  return this.options.name + ':cache:' + hash(obj);
};

// Sets a value in all stores, no manipulation of value at all
const multiSet = function(stores, key, value) {
  return Promise.map(stores,
                     function(store) {
                       return store.set(key, value);
                     })
    .then(function() {
      return value;
    })
    .catch(function() {
      return value;
    });
};

Cache.prototype.set = function set(key, value) {
  return multiSet(this.stores, key, value);
};

const processSearchResult = function(search) {
  // If we found it, return
  if(search.found) {
    return search.value;
  } else {
    // Otherwise throw a not found
    throw new NotFoundError(search.key);
  }
};

// Returns raw data out of the stores, repopulating stores that don't have the
// data from ones that do.
Cache.prototype.get = function get(key) {
  let self = this;

  // Accumulator object
  let search = {
    found: false,
    stores: [],
    key: key
  };

  return Promise.reduce(
    this.stores,
    function(search, store) {
      // If we've found the item already, just return
      if(search.found) {
        return search;
      }

      // Otherwise check this store
      return store.get(key).then(function(value) {
        // After this function returns,
        // set this value in the failed stores
        process.nextTick(function() {
          self.lastPromise = multiSet(search.stores, key, value);
        });

        // We found it!
        search.found = true;
        search.value = value;
        return search;

      }).catch(function(err) {
        // Error or not found, just keep looking
        search.stores.push(store);
        return search;
      });
    },
    search)
    .then(processSearchResult);
};

// Puts the raw data into a Value object and then sets it in the stores
Cache.prototype.createValueAndMultiSet = function(key, data, opts = {}) {
  let value = new Value(data);

  opts = Object.assign({}, this.options, opts);

  value.setStaleTTL(opts.staleTTL);
  value.setExpireTTL(opts.expireTTL);

  return multiSet(this.stores, key, JSON.stringify(value))
    .then(function() {
      return value;
    });
};

// Calls the function, storing the response
Cache.prototype.refresh = function(key, func, options = {}) {
  let self = this;
  return Promise.try(func).then(function(data) {
    return self.createValueAndMultiSet(key, data, options);
  }).then(function(value) {
    return value.get();
  });
};

// Wraps a function with the caching handling stale and expired states correctly
Cache.prototype.wrap = function wrap(key, func, options = {}) {
  let self = this;

  key = this.getKey(key);

  // First try and get the key
  return this.get(key).then(function(raw) {
    let value = Value.fromJSON(raw);

    if(value.expired()) {
      // Expire values wait for us to get them again, throwing errors
      return self.refresh(key, func, options);

    } else if(value.stale()) {
      // Stale values update on next tick
      process.nextTick(function() {
        self.lastPromise = Promise.try(func).then(function(data) {
          return self.createValueAndMultiSet(key, data, options);
        }).catch(function(err) {
          // Error getting new value... do nothing
        });
      });
    }

    // As long as not expired, return it!
    return value.get();
  }).catch(function(err) {
    // We couldn't find the value in the cache.
    // Run the function and then set it
    return self.refresh(key, func, options);
  });
};

const addContent = function addContent(cache, content, encoding) {
  if (Buffer.isBuffer(content)) {
    var oldContent = cache.content
    if (!oldContent) {
      oldContent = Buffer.alloc(0);
    }
    cache.content = Buffer.concat([oldContent, content], oldContent.length + content.length);
  } else {
    if(typeof content !== "undefined") {
      cache.content = (cache.content || '') + content;
    }
  }
  cache.encoding = encoding || cache.encoding;
};

// A connect middleware that supports stale and expired correctly
Cache.prototype.middleware = function middleware(opts = {}) {
  let self = this;
  opts = Object.assign({}, this.options, opts);

  return function(req, res, next) {
    // Bypass if we supplied header
    if(req.headers[opts.bypassHeader]) {
      res.setHeader(opts.statusHeader, 'bypass');
      return next();
    }

    if(opts.includeMethods.indexOf(req.method) === -1) {
      res.setHeader(opts.statusHeader, 'skipMethod');<commit_msg>Protection from old content not being a buffer<commit_after>// Main Cache Interface
const Promise = require('bluebird');
const hash = require('object-hash');
const moment = require('moment');
const _ = require('lodash');

const Value = require('./value');
const NotFoundError = require('./errors/notFound');

const defaultOptions = {
  name: 'default',
  bypassHeader: 'cache-bypass',
  statusHeader: 'cache-status',
  includeMethods: ['GET']
};

/**
 * A cache object
 *
 * Takes in an array of stores and a hash of options
 **/
function Cache(stores = [], options = {}) {
  if(!(stores instanceof Array)) {
    stores = [stores];
  }

  this.stores = stores;
  this.options = Object.assign({},
                               defaultOptions,
                               options);
};

// Returns the list of stores
Cache.prototype.getStores = function getStores() {
  return this.stores;
};

// A key generator based on [object-hash](https://github.com/puleos/object-hash)
Cache.prototype.getKey = function getKey(obj = {}) {
  return this.options.name + ':cache:' + hash(obj);
};

// Sets a value in all stores, no manipulation of value at all
const multiSet = function(stores, key, value) {
  return Promise.map(stores,
                     function(store) {
                       return store.set(key, value);
                     })
    .then(function() {
      return value;
    })
    .catch(function() {
      return value;
    });
};

Cache.prototype.set = function set(key, value) {
  return multiSet(this.stores, key, value);
};

const processSearchResult = function(search) {
  // If we found it, return
  if(search.found) {
    return search.value;
  } else {
    // Otherwise throw a not found
    throw new NotFoundError(search.key);
  }
};

// Returns raw data out of the stores, repopulating stores that don't have the
// data from ones that do.
Cache.prototype.get = function get(key) {
  let self = this;

  // Accumulator object
  let search = {
    found: false,
    stores: [],
    key: key
  };

  return Promise.reduce(
    this.stores,
    function(search, store) {
      // If we've found the item already, just return
      if(search.found) {
        return search;
      }

      // Otherwise check this store
      return store.get(key).then(function(value) {
        // After this function returns,
        // set this value in the failed stores
        process.nextTick(function() {
          self.lastPromise = multiSet(search.stores, key, value);
        });

        // We found it!
        search.found = true;
        search.value = value;
        return search;

      }).catch(function(err) {
        // Error or not found, just keep looking
        search.stores.push(store);
        return search;
      });
    },
    search)
    .then(processSearchResult);
};

// Puts the raw data into a Value object and then sets it in the stores
Cache.prototype.createValueAndMultiSet = function(key, data, opts = {}) {
  let value = new Value(data);

  opts = Object.assign({}, this.options, opts);

  value.setStaleTTL(opts.staleTTL);
  value.setExpireTTL(opts.expireTTL);

  return multiSet(this.stores, key, JSON.stringify(value))
    .then(function() {
      return value;
    });
};

// Calls the function, storing the response
Cache.prototype.refresh = function(key, func, options = {}) {
  let self = this;
  return Promise.try(func).then(function(data) {
    return self.createValueAndMultiSet(key, data, options);
  }).then(function(value) {
    return value.get();
  });
};

// Wraps a function with the caching handling stale and expired states correctly
Cache.prototype.wrap = function wrap(key, func, options = {}) {
  let self = this;

  key = this.getKey(key);

  // First try and get the key
  return this.get(key).then(function(raw) {
    let value = Value.fromJSON(raw);

    if(value.expired()) {
      // Expire values wait for us to get them again, throwing errors
      return self.refresh(key, func, options);

    } else if(value.stale()) {
      // Stale values update on next tick
      process.nextTick(function() {
        self.lastPromise = Promise.try(func).then(function(data) {
          return self.createValueAndMultiSet(key, data, options);
        }).catch(function(err) {
          // Error getting new value... do nothing
        });
      });
    }

    // As long as not expired, return it!
    return value.get();
  }).catch(function(err) {
    // We couldn't find the value in the cache.
    // Run the function and then set it
    return self.refresh(key, func, options);
  });
};

const addContent = function addContent(cache, content, encoding) {
  if (Buffer.isBuffer(content)) {
    var oldContent = Buffer.from(cache.content || '');
    cache.content = Buffer.concat([oldContent, content], oldContent.length + content.length);
  } else {
    if(typeof content !== "undefined") {
      cache.content = (cache.content || '') + content;
    }
  }
  cache.encoding = encoding || cache.encoding;
};

// A connect middleware that supports stale and expired correctly
Cache.prototype.middleware = function middleware(opts = {}) {
  let self = this;
  opts = Object.assign({}, this.options, opts);

  return function(req, res, next) {
    // Bypass if we supplied header
    if(req.headers[opts.bypassHeader]) {
      res.setHeader(opts.statusHeader, 'bypass');
      return next();
    }

    if(opts.includeMethods.indexOf(req.method) === -1) {
      res.setHeader(opts.statusHeader, 'skipMethod');
{'fraction_non_alphanumeric': 0.13268844679686073, 'fraction_numerical': 0.0002737725862383647, 'mean_word_length': 2.7288193263014633, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3893312', 'n_tokens_mistral': 3258, 'n_tokens_neox': 3205, 'n_words': 916}