<commit_before>'use strict';

const { name, version, homepage } = require('./package');

const urlParser = require('url');
const uuid = require('uuid/v1');
const moment = require('moment');
const debug = require('debug')(name);

const { parseRequestCookies, parseResponseCookies } = require('./lib/cookies');
const {
  calculateRequestHeaderSize,
  calculateResponseHeaderSize,
  getHeaderValue,
  parseHeaders
} = require('./lib/headers');

const max = Math.max;

const defaultOptions = {
  includeResourcesFromDiskCache: false
};

const isEmpty = o => !o;

function formatMillis(time, fractionalDigits = 3) {
  return Number(Number(time).toFixed(fractionalDigits));
}

function formatIP(ipAddress) {
  if (typeof ipAddress !== 'string') {
    return undefined;
  }
  // IPv6 addresses are listed as [fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b]
  return ipAddress.replace(/^\[|]$/g, '');
}

function populateEntryFromResponse(entry, response, page) {
  const responseHeaders = response.headers;
  const cookieHeader = getHeaderValue(responseHeaders, 'Set-Cookie');

  entry.response = {
    httpVersion: response.protocol,
    redirectURL: '',
    status: response.status,
    statusText: response.statusText,
    content: {
      mimeType: response.mimeType,
      size: 0
    },
    headersSize: -1,
    bodySize: -1,
    cookies: parseResponseCookies(cookieHeader),
    headers: parseHeaders(responseHeaders),
    _transferSize: response.encodedDataLength
  };

  const locationHeaderValue = getHeaderValue(responseHeaders, 'Location');
  if (locationHeaderValue) {
    entry.response.redirectURL = locationHeaderValue;
  }

  entry.request.httpVersion = response.protocol;

  if (response.fromDiskCache === true) {
    if (isHttp1x(response.protocol)) {
      // In http2 headers are compressed, so calculating size from headers text wouldn't be correct.
      entry.response.headersSize = calculateResponseHeaderSize(response);
    }

    // h2 push might cause resource to be received before parser sees and requests it.
    if (!(response.pushStart > 0)) {
      entry.cache.beforeRequest = {
        lastAccess: '',
        eTag: '',<commit_msg>Fixed missing pushed objects in the HAR file (#21)<commit_after>'use strict';

const { name, version, homepage } = require('./package');

const urlParser = require('url');
const uuid = require('uuid/v1');
const moment = require('moment');
const debug = require('debug')(name);

const { parseRequestCookies, parseResponseCookies } = require('./lib/cookies');
const {
  calculateRequestHeaderSize,
  calculateResponseHeaderSize,
  getHeaderValue,
  parseHeaders
} = require('./lib/headers');

const max = Math.max;

const defaultOptions = {
  includeResourcesFromDiskCache: false
};

const isEmpty = o => !o;

function formatMillis(time, fractionalDigits = 3) {
  return Number(Number(time).toFixed(fractionalDigits));
}

function formatIP(ipAddress) {
  if (typeof ipAddress !== 'string') {
    return undefined;
  }
  // IPv6 addresses are listed as [fd00:a516:7c1b:17cd:6d81:2137:bd2a:2c5b]
  return ipAddress.replace(/^\[|]$/g, '');
}

function populateEntryFromResponse(entry, response, page) {
  const responseHeaders = response.headers;
  const cookieHeader = getHeaderValue(responseHeaders, 'Set-Cookie');

  entry.response = {
    httpVersion: response.protocol,
    redirectURL: '',
    status: response.status,
    statusText: response.statusText,
    content: {
      mimeType: response.mimeType,
      size: 0
    },
    headersSize: -1,
    bodySize: -1,
    cookies: parseResponseCookies(cookieHeader),
    headers: parseHeaders(responseHeaders),
    _transferSize: response.encodedDataLength
  };

  const locationHeaderValue = getHeaderValue(responseHeaders, 'Location');
  if (locationHeaderValue) {
    entry.response.redirectURL = locationHeaderValue;
  }

  entry.request.httpVersion = response.protocol;

  if (response.fromDiskCache === true) {
    if (isHttp1x(response.protocol)) {
      // In http2 headers are compressed, so calculating size from headers text wouldn't be correct.
      entry.response.headersSize = calculateResponseHeaderSize(response);
    }

    // h2 push might cause resource to be received before parser sees and requests it.
    if (!(response.timing.pushStart > 0)) {
      entry.cache.beforeRequest = {
        lastAccess: '',
        eTag: '',
{'fraction_non_alphanumeric': 0.12241141279337321, 'fraction_numerical': 0.013805798435342844, 'mean_word_length': 4.19976076555024, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4948287', 'n_tokens_mistral': 1309, 'n_tokens_neox': 1258, 'n_words': 302}