<commit_before>var inherits = require('util').inherits;
var events = require('events');
var crypto = require('crypto');
var fs = require('fs');

var DEFAULT_STORAGE = 'memory';
var DEFAULT_WRITE_PERMS = '0644';
var DEFAULT_INDEX_LIMIT = 1000;

/**
 * Cache constructor
 *
 * @param  string clientId
 * @param  array options
 */
var Cache = function(clientId, options) {
  events.EventEmitter.call(this);

  this.versions = null;
  this.indexes = null;
  this.memory = {};

  options = options || {};
  if (typeof options === 'string') {
    options = { path: options };
  }
  this.params = {
    clientId: clientId,
    path: options.path ? String(options.path) : '',
    storage: options.storage || DEFAULT_STORAGE,
    writePerms: options.writePerms || DEFAULT_WRITE_PERMS,
    indexLimit: options.indexLimit || DEFAULT_INDEX_LIMIT
  };

  if (this.params.storage !== 'memory') {
    throw new Error(this.params.storage + ' storage is not currently supported');
  }
};

inherits(Cache, events.EventEmitter);

/**
 * Get result from cache by url/data
 *
 * @param  string url
 * @param  mixed data
 */
Cache.prototype.get = function(url, data) {
  data = data || null;

  var cacheKey = this.getKey(url, data);
  var result = this.getCache(cacheKey, 'result');

  if (result) {
    // Ensure cache_key exists in index
    this.getIndex();
    if (result.$collection !== undefined) {
      var collection = result.$collection;
      if (this.indexes[collection] && this.indexes[collection][cacheKey]) {
        return result;
      }
    }

    // Not found in proper index, then clear?
    var resultCollections = this.resultCollections(result);
    for (var i = 0; i < resultCollections.length; i++) {
      var collection = resultCollections[i];
      var where = {};
      where[collection] = cacheKey;
      this.clearIndexes(where);
    }
  }

  return null;
};

/**
 * Get a cache key
 *
 * @param  string url
 * @param  mixed data
 * @return string
 */
Cache.prototype.getKey = function(url, data) {
  data = data || null;
  var saneUrl = String(url).trim().replace(/^\/|\/$/g, '');
  var keyData = JSON.stringify([ saneUrl, data ]);
  return crypto.createHash('md5').update(keyData).digest('hex');
};

/**
 * Get path to a cache file
 *
 * @return string
 */
Cache.prototype.getPath = function(url, data) {
  return this.params.path.replace(/\/$/, '') +
    '/client.' +
    this.params.clientId +
    '.' +
    Array.prototype.slice.call(arguments).join('.');
};

/**
 * Get cache version info
 *
 * @return array
 */
Cache.prototype.getVersions = function() {
  if (!this.versions) {
    this.versions = this.getCache('versions') || {};
  }
  return this.versions;
};

/**
 * Get cache index info
 *
 * @return array
 */
Cache.prototype.getIndex = function() {
  if (!this.indexes) {
    this.indexes = this.getCache('index') || {};
  }
  return this.indexes;
};

/**
 * Put cache result in storage atomicly
 *
 * @param  string url
 * @param  mixed data
 * @param  mixed result
 */
Cache.prototype.put = function(url, data, result) {
  if (result.$data === undefined) {
    result.$data = null; // Allows for null response
  }

  this.getVersions();

  var cacheContent = Object.assign({}, result);
  cacheContent.$cached = true;

  var cacheKey = this.getKey(url, data);
  var cachePath = this.getPath(cacheKey, 'result');

  var size = this.writeCache(cachePath, cacheContent);

  if (size > 0) {
    if (result.$cached !== undefined) {
      var cached = result.$cached;
      var resultCollections = this.resultCollections(result);
      for (var i = 0; i < resultCollections.length; i++) {
        var collection = resultCollections[i];
        // Collection may not be cacheable
        if (cached[collection] === undefined && this.versions[collection] === undefined) {
          continue;
        }
        this.putIndex(collection, cacheKey, size);
        if (cached[collection] !== undefined) {
          this.putVersion(collection, cached[collection]);
        }
      }
    }
  }
};<commit_msg>Fix for earlier versions of node<commit_after>var inherits = require('util').inherits;
var events = require('events');
var crypto = require('crypto');
var fs = require('fs');

var DEFAULT_STORAGE = 'memory';
var DEFAULT_WRITE_PERMS = '0644';
var DEFAULT_INDEX_LIMIT = 1000;

/**
 * Cache constructor
 *
 * @param  string clientId
 * @param  array options
 */
var Cache = function(clientId, options) {
  events.EventEmitter.call(this);

  this.versions = null;
  this.indexes = null;
  this.memory = {};

  options = options || {};
  if (typeof options === 'string') {
    options = { path: options };
  }
  this.params = {
    clientId: clientId,
    path: options.path ? String(options.path) : '',
    storage: options.storage || DEFAULT_STORAGE,
    writePerms: options.writePerms || DEFAULT_WRITE_PERMS,
    indexLimit: options.indexLimit || DEFAULT_INDEX_LIMIT
  };

  if (this.params.storage !== 'memory') {
    throw new Error(this.params.storage + ' storage is not currently supported');
  }
};

inherits(Cache, events.EventEmitter);

/**
 * Get result from cache by url/data
 *
 * @param  string url
 * @param  mixed data
 */
Cache.prototype.get = function(url, data) {
  data = data || null;

  var cacheKey = this.getKey(url, data);
  var result = this.getCache(cacheKey, 'result');

  if (result) {
    // Ensure cache_key exists in index
    this.getIndex();
    if (result.$collection !== undefined) {
      var collection = result.$collection;
      if (this.indexes[collection] && this.indexes[collection][cacheKey]) {
        return result;
      }
    }

    // Not found in proper index, then clear?
    var resultCollections = this.resultCollections(result);
    for (var i = 0; i < resultCollections.length; i++) {
      var collection = resultCollections[i];
      var where = {};
      where[collection] = cacheKey;
      this.clearIndexes(where);
    }
  }

  return null;
};

/**
 * Get a cache key
 *
 * @param  string url
 * @param  mixed data
 * @return string
 */
Cache.prototype.getKey = function(url, data) {
  data = data || null;
  var saneUrl = String(url).trim().replace(/^\/|\/$/g, '');
  var keyData = JSON.stringify([ saneUrl, data ]);
  return crypto.createHash('md5').update(keyData).digest('hex');
};

/**
 * Get path to a cache file
 *
 * @return string
 */
Cache.prototype.getPath = function(url, data) {
  return this.params.path.replace(/\/$/, '') +
    '/client.' +
    this.params.clientId +
    '.' +
    Array.prototype.slice.call(arguments).join('.');
};

/**
 * Get cache version info
 *
 * @return array
 */
Cache.prototype.getVersions = function() {
  if (!this.versions) {
    this.versions = this.getCache('versions') || {};
  }
  return this.versions;
};

/**
 * Get cache index info
 *
 * @return array
 */
Cache.prototype.getIndex = function() {
  if (!this.indexes) {
    this.indexes = this.getCache('index') || {};
  }
  return this.indexes;
};

/**
 * Put cache result in storage atomicly
 *
 * @param  string url
 * @param  mixed data
 * @param  mixed result
 */
Cache.prototype.put = function(url, data, result) {
  if (result.$data === undefined) {
    result.$data = null; // Allows for null response
  }

  this.getVersions();

  var cacheContent = {};
  var keys = Object.keys(result);
  for (var i = 0; i < keys.length; i++) {
    cacheContent[keys[i]] = result[keys[i]];
  }
  cacheContent.$cached = true;

  var cacheKey = this.getKey(url, data);
  var cachePath = this.getPath(cacheKey, 'result');

  var size = this.writeCache(cachePath, cacheContent);

  if (size > 0) {
    if (result.$cached !== undefined) {
      var cached = result.$cached;
      var resultCollections = this.resultCollections(result);
      for (var i = 0; i < resultCollections.length; i++) {
        var collection = resultCollections[i];
        // Collection may not be cacheable
        if (cached[collection] === undefined && this.versions[collection] === undefined) {
          continue;
        }
        this.putIndex(collection, cacheKey, size);
        if (cached[collection] !== undefined) {
          this.putVersion(collection, cached[collection]);
        }
      }
    }
  }
};
{'fraction_non_alphanumeric': 0.153487801887949, 'fraction_numerical': 0.0030648522741203873, 'mean_word_length': 3.0871743486973946, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4571798', 'n_tokens_mistral': 2544, 'n_tokens_neox': 2511, 'n_words': 631}