<commit_before>var http = require('http');
var dns = require('dns');
var url  = require('url');
var clone  = require('clone');
var httpProxy = require('http-proxy');
var proxy = new httpProxy.RoutingProxy();
var redis  = require('redis'),
    client = redis.createClient(6379, '127.0.0.1', {detect_buffers: true});
var events = require('events'),
  eventEmitter = new events.EventEmitter();


function computeKey(host, path, method) {
  return method + ':' + host + ':' + path;
}

function makeKeyField(key, prefix) {
  return prefix + ':' + key;
}

function getHeadersKey(key) {
  return makeKeyField(key, 'headers');
}

function getBodyKey(key) {
  return makeKeyField(key, 'body');
}

function canStore(statusCode, headers) {
  // Do not cache 304 response to conditional requests
  if( isConditional(headers) && statusCode == 304 ) {
    return false;
  }

  // rules are
  // if not private
  // and not no-store nor no cache
  // then use the value in this order
  // s-maxage
  // max-age
  // expires
  // Exceptions are 
  // must-revalidate
  // proxy-revalidate

  if( headers['cache-control'] === undefined ) {
    return false;
  }

  var params = headers['cache-control'].trim().split(',');

  var values = {};

  for(var idx in params) {
    if( params[idx].indexOf('=') !== -1 ) {
      var parts = params[idx].trim().split('=');
      key = parts[0].trim();
      val = parts[1].trim();
      if( key == 's-maxage' || key == 'max-age' ) {
        val = parseInt(val);
      }
      values[ key ] = val;
    } else {
      values[ params[idx].trim() ] = 1;
    }
  }

  if( 
      values.private !== undefined ||
      values['no-cache'] !== undefined ||
      values['no-store'] !== undefined 
    ) {
      // Do not cache
      return false;
    }

  if( values['s-maxage'] !== undefined ) {
    return values['s-maxage'];
  }
  else if( values['max-age'] !== undefined ) {
    return values['max-age'];
  }

  return false;
}

function isNotModified(req, resp) {
  if( req['if-modified-since'] !== undefined && 
      resp['last-modified'] !== undefined ) {
        var condDate = new Date(req['if-modified-since']);
        var lastModified = new Date(resp['last-modified']);
        return condDate >= lastModified;<commit_msg>Refactor parsing of cache-control header<commit_after>var http = require('http');
var dns = require('dns');
var url  = require('url');
var clone  = require('clone');
var httpProxy = require('http-proxy');
var proxy = new httpProxy.RoutingProxy();
var redis  = require('redis'),
    client = redis.createClient(6379, '127.0.0.1', {detect_buffers: true});
var events = require('events'),
  eventEmitter = new events.EventEmitter();


function computeKey(host, path, method) {
  return method + ':' + host + ':' + path;
}

function makeKeyField(key, prefix) {
  return prefix + ':' + key;
}

function getHeadersKey(key) {
  return makeKeyField(key, 'headers');
}

function getBodyKey(key) {
  return makeKeyField(key, 'body');
}

function splitString(str, separator) {
  return str.trim().split(separator);
}

function getHeaderValues(str) {
  var params = splitString(str, ',');

  var values = {};

  for(var idx in params) {
    var parts = params[idx].trim().split('=');
    if( parts.length > 0 ) {
      var key = parts[0].trim();
      if( parts.length >= 2 ) {
        val = parts[1].trim();
        values[ key ] = val;
      } else {
        values[ key ] = true;
      }
    }
  }

  return values;
}

function canStore(statusCode, headers) {
  // Do not cache 304 response to conditional requests
  if( isConditional(headers) && statusCode == 304 ) {
    return false;
  }

  // rules are
  // if not private
  // and not no-store nor no cache
  // then use the value in this order
  // s-maxage
  // max-age
  // expires
  // Exceptions are 
  // must-revalidate
  // proxy-revalidate

  if( headers['cache-control'] === undefined ) {
    return false;
  }

  var values = getHeaderValues(headers['cache-control']);

  if( 
      values.private !== undefined ||
      values['no-cache'] !== undefined ||
      values['no-store'] !== undefined 
    ) {
      // Do not cache
      return false;
    }

  if( values['s-maxage'] !== undefined ) {
    return parseInt(values['s-maxage']);
  }
  else if( values['max-age'] !== undefined ) {
    return parseInt(values['max-age']);
  }

  return false;
}

function isNotModified(req, resp) {
  if( req['if-modified-since'] !== undefined && 
      resp['last-modified'] !== undefined ) {
        var condDate = new Date(req['if-modified-since']);
        var lastModified = new Date(resp['last-modified']);
        return condDate >= lastModified;
{'fraction_non_alphanumeric': 0.16799311679931167, 'fraction_numerical': 0.008604000860400087, 'mean_word_length': 2.9608177172061327, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '279716', 'n_tokens_mistral': 1513, 'n_tokens_neox': 1464, 'n_words': 374}