<commit_before>
var util = require("util");
var events = require("events");
var nodeshark = require('./nodeShark');

module.exports = HttpTracker = function(tcpTracker) {
  tcpTracker.on("packet", this._onPacket.bind(this));
  events.EventEmitter.call(this);
}
util.inherits(HttpTracker, events.EventEmitter);

HttpTracker.prototype._onPacket = function(session, packet) {
  var httpTrackerSession = session.httpTrackerSession;

  if(!httpTrackerSession && !packet.http) {
    return;
  }

  // request
  if(packet.http && packet.http.request && packet.http.request.value == 1) {
    httpTrackerSession = {
      getTcpTrackerSession: function() { return session; },
      request: {
        headers: {

        }
      },
      response: {
        headers: {

        }
      }
    };
    session.httpTrackerSession = httpTrackerSession;

    this._populateRequestOrResponse(httpTrackerSession.request, packet);
    this.emit("requestHeaders", httpTrackerSession);

    if(packet["data-text-lines"]) {
      this.emit("requestData", httpTrackerSession, packet["data-text-lines"]);
    }
  }

  // response
  if(httpTrackerSession && packet.http && packet.http.response && packet.http.response.value == 1) {
    this._populateRequestOrResponse(httpTrackerSession.response, packet);
    this.emit("responseHeaders", httpTrackerSession);
    if(packet["data-text-lines"]) {
      this.emit("responseData", httpTrackerSession, packet["data-text-lines"]);
    }
  }
}

HttpTracker.prototype._populateRequestOrResponse = function(requestResponse, packet) {
  for(var key in packet.http) {
    if(nodeshark.isStandardKey(key)) continue;
    if(key == '\\r\\n') continue;
    if(key == 'request') continue;
    if(key == 'response') continue;
    if(key == 'request.full_uri') {
      requestResponse.fullUri = packet.http[key].value;
      continue;
    }
    if(packet.http[key]["http.request.method"]) {
      requestResponse.method = packet.http[key]["http.request.method"].value;
      requestResponse.uri = packet.http[key]["http.request.uri"].value;
      requestResponse.version = packet.http[key]["http.request.version"].value;
      continue;
    }
    if(packet.http[key]["http.response.code"]) {
      requestResponse.codeString = packet.http[key]["http.response.phrase"].value;
      requestResponse.code = packet.http[key]["http.response.code"].value;
      requestResponse.version = packet.http[key]["http.request.version"].value;
      continue;
    }
    if(key.match(/\r\n$/) == "\r\n") {
      var shortKey = key.slice(0, key.length-2);
      var token = shortKey.indexOf(':');
      if(token > 0) {
        var name = shortKey.slice(0, token);
        var value = shortKey.slice(token+2);
        requestResponse.headers[name] = value;
        continue;
      }
    }
    requestResponse.headers[this._headerKeyToDisplay(key)] = packet.http[key].value;
  }
}


HttpTracker.prototype._headerKeyToDisplay = function(key) {
  if(key == 'host') return 'Host';
  if(key == 'user_agent') return 'User-Agent';
  if(key == 'accept') return 'Accept';
  if(key == 'accept_language') return 'Accept-Language';
  if(key == 'accept_encoding') return 'Accept-Encoding';
  if(key == 'connection') return 'Connection';
  if(key == 'referer') return 'Referer';
  if(key == 'date') return 'Date';
  if(key == 'server') return 'Server';
  if(key == 'last_modified') return 'Last-Modified';
  if(key == 'content_length_header') return 'Content-Length';
  if(key == 'content_type') return 'Content-Type';
  return key;
}
<commit_msg>emit responseData event even if we don't really have data<commit_after>
var util = require("util");
var events = require("events");
var nodeshark = require('./nodeShark');

module.exports = HttpTracker = function(tcpTracker) {
  tcpTracker.on("packet", this._onPacket.bind(this));
  events.EventEmitter.call(this);
}
util.inherits(HttpTracker, events.EventEmitter);

HttpTracker.prototype._onPacket = function(session, packet) {
  var httpTrackerSession = session.httpTrackerSession;

  if(!httpTrackerSession && !packet.http) {
    return;
  }

  // request
  if(packet.http && packet.http.request && packet.http.request.value == 1) {
    httpTrackerSession = {
      getTcpTrackerSession: function() { return session; },
      request: {
        headers: {

        }
      },
      response: {
        headers: {

        }
      }
    };
    session.httpTrackerSession = httpTrackerSession;

    this._populateRequestOrResponse(httpTrackerSession.request, packet);
    this.emit("requestHeaders", httpTrackerSession);

    if(packet["data-text-lines"]) {
      this.emit("requestData", httpTrackerSession, packet["data-text-lines"]);
    }
  }

  // response
  if(httpTrackerSession && packet.http && packet.http.response && packet.http.response.value == 1) {
    this._populateRequestOrResponse(httpTrackerSession.response, packet);
    this.emit("responseHeaders", httpTrackerSession);
    if(packet["data-text-lines"]) {
      this.emit("responseData", httpTrackerSession, packet["data-text-lines"]);
    } else {
      this.emit("responseData", httpTrackerSession);
    }
  }
}

HttpTracker.prototype._populateRequestOrResponse = function(requestResponse, packet) {
  for(var key in packet.http) {
    if(nodeshark.isStandardKey(key)) continue;
    if(key == '\\r\\n') continue;
    if(key == 'request') continue;
    if(key == 'response') continue;
    if(key == 'request.full_uri') {
      requestResponse.fullUri = packet.http[key].value;
      continue;
    }
    if(packet.http[key]["http.request.method"]) {
      requestResponse.method = packet.http[key]["http.request.method"].value;
      requestResponse.uri = packet.http[key]["http.request.uri"].value;
      requestResponse.version = packet.http[key]["http.request.version"].value;
      continue;
    }
    if(packet.http[key]["http.response.code"]) {
      requestResponse.codeString = packet.http[key]["http.response.phrase"].value;
      requestResponse.code = packet.http[key]["http.response.code"].value;
      requestResponse.version = packet.http[key]["http.request.version"].value;
      continue;
    }
    if(key.match(/\r\n$/) == "\r\n") {
      var shortKey = key.slice(0, key.length-2);
      var token = shortKey.indexOf(':');
      if(token > 0) {
        var name = shortKey.slice(0, token);
        var value = shortKey.slice(token+2);
        requestResponse.headers[name] = value;
        continue;
      }
    }
    requestResponse.headers[this._headerKeyToDisplay(key)] = packet.http[key].value;
  }
}


HttpTracker.prototype._headerKeyToDisplay = function(key) {
  if(key == 'host') return 'Host';
  if(key == 'user_agent') return 'User-Agent';
  if(key == 'accept') return 'Accept';
  if(key == 'accept_language') return 'Accept-Language';
  if(key == 'accept_encoding') return 'Accept-Encoding';
  if(key == 'connection') return 'Connection';
  if(key == 'referer') return 'Referer';
  if(key == 'date') return 'Date';
  if(key == 'server') return 'Server';
  if(key == 'last_modified') return 'Last-Modified';
  if(key == 'content_length_header') return 'Content-Length';
  if(key == 'content_type') return 'Content-Type';
  return key;
}

{'fraction_non_alphanumeric': 0.16061452513966482, 'fraction_numerical': 0.0019553072625698325, 'mean_word_length': 3.7049934296977662, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4467871', 'n_tokens_mistral': 2211, 'n_tokens_neox': 2193, 'n_words': 370}