<commit_before>var http = require('http');

var ADDITIONAL_CODE = "<script type='text/javascript' src='http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js'></script>" +
                      "<script type='text/javascript'>$.noConflict();</script>" +
                      "<script type='text/javascript' src='http://cdnjs.cloudflare.com/ajax/libs/require.js/1.0.1/require.min.js'></script>" +
                      "<script type='text/javascript' src='http://axiak.github.com/injectfun/index.js'></script>";

var PORT = process.env.PORT || 8000;

console.log("Proxy listening on port " + PORT);

function fixHeaders(oldHeaders) {
  // node does something STUPID in that incoming headers will be all lowercased
  // but outgoing headers will not have their case affected so I have to fix
  // them here.
  // Return a new hash of HTTP headers such that each header name (key) in this
  // hash has the proper case. This will not work for the "TE" header, see
  // http://en.wikipedia.org/wiki/List_of_HTTP_header_fields
  var result = {};
  for (var header in oldHeaders) {
    if (oldHeaders.hasOwnProperty(header)) {(function(){
      // this is jslint's idea of "code style" ^ ^ ^
      // (personally, I think this entire function is full of fail. thanks,
      // node. thanks, jslint.)
      header = header.split('-')
                     .map(function(header){ return header[0].toUpperCase()+header.slice(1); })
                     .join('-');
      result[header] = oldHeaders[header.toLowerCase()];
    }());}
  }
  return result;
}

http.createServer(function(request, response) {
//  request.headers['accept-encoding'] = 'identity';
  delete request.headers['accept-encoding'];
  delete request.headers['proxy-connection'];

  var proxy = http.createClient(80, request.headers['host']);

  var proxy_request = proxy.request(request.method, request.url, fixHeaders(request.headers));

  proxy_request.addListener('response', function (proxy_response) {
    var isHtml = (proxy_response.headers['content-type'] &&
                  proxy_response.headers['content-type'].toLowerCase().indexOf("html") != -1),
        buffer = "";

    proxy_response.addListener('error', function (error) {
        console.log(error);
        console.log(error.stack);
    });

    proxy_response.addListener('data', function(chunk) {
      if (isHtml) {
          buffer += chunk.toString("utf-8");
      } else {
          response.write(chunk, 'binary');
      }
    });

    proxy_response.addListener('end', function() {
      if (isHtml) {
          delete proxy_response.headers['content-length'];
          var originalLength = buffer.length;
          buffer = buffer.replace(/<\/body>/i, ADDITIONAL_CODE + "</body>");
          if (buffer.length == originalLength && buffer.search(/<html>/i) !== -1) {
              buffer += ADDITIONAL_CODE;
          }
          response.end(buffer);
      } else {
          response.end();
      }
    });
    response.writeHead(proxy_response.statusCode, proxy_response.headers);
  });
  request.addListener('data', function(chunk) {
    proxy_request.write(chunk, 'binary');
  });
  request.addListener('end', function() {
    proxy_request.end();
  });
}).listen(process.env.PORT || 8000);<commit_msg>Fixed proxy.<commit_after>var http = require('http')
  , url = require('url');
var ADDITIONAL_CODE = "<script type='text/javascript' src='http://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js'></script>" +
                      "<script type='text/javascript'>$.noConflict();</script>" +
                      "<script type='text/javascript' src='http://cdnjs.cloudflare.com/ajax/libs/require.js/1.0.1/require.min.js'></script>" +
                      "<script type='text/javascript' src='http://axiak.github.com/injectfun/index.js'></script>";

var PORT = process.env.PORT || 8000;

function fixHeaders(oldHeaders) {
  // node does something STUPID in that incoming headers will be all lowercased
  // but outgoing headers will not have their case affected so I have to fix
  // them here.
  // Return a new hash of HTTP headers such that each header name (key) in this
  // hash has the proper case. This will not work for the "TE" header, see
  // http://en.wikipedia.org/wiki/List_of_HTTP_header_fields
  var result = {};
  for (var header in oldHeaders) {
    if (oldHeaders.hasOwnProperty(header)) {(function(){
      header = header.split('-')
                     .map(function(header){ return header[0].toUpperCase()+header.slice(1); })
                     .join('-');
      result[header] = oldHeaders[header.toLowerCase()];
    }());}
  }
  return result;
}

var catch_errors = function (wrapped) {
    return function (request, response) {
        try {
            return wrapped(request, response);
        } catch (error) {
            console.log("CAUGHT");
            console.log(error);
            console.log(error.stack);
        }
    };
};

var unrecoverable_error = function (request, response) {
    return function (error) {
        console.log('------------------------------------------------');
        console.log("Unrecoverable error: " + error);
        console.log("URL: " + request.method + " " + request.url);
        console.log(error.stack);
        response.writeHead(404, {'X-Failure': ""+error});
        response.end('');
    };
};


var debug_view = function (request_info) {
    if (process.env.DEBUG) {
        var r = {};
        r['host'] = request_info['host'];
        r['port'] = request_info['port'];
        r['path'] = request_info['path'];
        r['method'] = request_info['method'];

        console.log(r);
    }
};


var server = http.createServer(catch_errors(function(request, response) {
  request.headers['accept-encoding'] = 'identity';
  delete request.headers['accept-encoding'];
  delete request.headers['proxy-connection'];

  var parsed_url = url.parse(request.url);

  var request_info = {
    'host': parsed_url.hostname
  , 'port': ~~(parsed_url.port || 80)
  , 'path': parsed_url.pathname + (parsed_url.search || '') + (parsed_url.hash || '')
  , 'method': request.method
  , 'headers': fixHeaders(request.headers)
  };

  debug_view(request_info);

  var proxy_request = http.request(request_info, function (proxy_response) {
    var isHtml = (proxy_response.headers['content-type'] &&
                  proxy_response.headers['content-type'].toLowerCase().indexOf("html") != -1),
        buffer = "";

    proxy_response.on('error', function (error) {
        console.log(error);
        console.log(error.stack);
    });

    proxy_response.on('data', function(chunk) {
      if (isHtml) {
          buffer += chunk.toString("utf-8");
      } else {
          response.write(chunk, 'binary');
      }
    });

    proxy_response.on('end', function() {
      if (isHtml) {
          delete proxy_response.headers['content-length'];
          var originalLength = buffer.length;
          buffer = buffer.replace(/<\/body>/i, ADDITIONAL_CODE + "</body>");
          if (buffer.length == originalLength && buffer.search(/<html>/i) !== -1) {
              buffer += ADDITIONAL_CODE;
          }
          response.end(buffer);
      } else {
          response.end();
      }
    });

    proxy_response.on('error', unrecoverable_error(request, response));

    response.writeHead(proxy_response.statusCode, proxy_response.headers);
  });

  proxy_request.on('error', unrecoverable_error(request, response));
  request.on('error', unrecoverable_error(request, response));

  request.on('data', function(chunk) {
    proxy_request.write(chunk, 'binary');
  });

  request.on('end', function() {
    proxy_request.end();
  });


}));

server.listen(PORT);
server.on('clientError', function (error) {
    console.log("ClientError failure.");
    console.log(error);
    console.log(error.stack);
});
server.on('error', function (error) {
    console.log("error failure.");
    console.log(error);
    console.log(error.stack);
});

console.log("Proxy listening on port " + PORT);
{'fraction_non_alphanumeric': 0.16787228735345472, 'fraction_numerical': 0.0051134946370666005, 'mean_word_length': 2.825858778625954, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5748468', 'n_tokens_mistral': 2388, 'n_tokens_neox': 2344, 'n_words': 471}