<commit_before>/**
 * Simple HTTP requests monitor and analyzer
 */
"use strict";

const assert = require("assert"),
  debug = require("debug")("phantomas:modules:requestsMonitor");

/**
 * Given key-value set of HTTP headers returns the set with lowercased header names
 *
 * @param {object} headers
 * @returns {object}
 */
function lowerCaseHeaders(headers) {
  var res = {};

  Object.keys(headers).forEach((headerName) => {
    res[headerName.toLowerCase()] = headers[headerName];
  });

  return res;
}

// parse given URL to get protocol and domain
function parseEntryUrl(entry) {
  var parsed;

  // asset type
  entry.type = "other";

  if (entry.url.indexOf("data:") === 0) {
    // base64 encoded data
    entry.domain = false;
    entry.protocol = false;
    entry.isBase64 = true;
  } else if (entry.url.indexOf("blob:") === 0) {
    // blob image or video
    entry.domain = false;
    entry.protocol = false;
    entry.isBlob = true;
  } else {
    parsed = new URL(entry.url) || {};

    entry.protocol = parsed.protocol.replace(":", ""); // e.g. "http:"
    entry.domain = parsed.hostname;
    entry.query = parsed.search.substring(1);

    if (entry.protocol === "https") {
      entry.isSSL = true;
    }
  }

  return entry;
}

/**
 * Detect response content type using "Content-Type header value"
 *
 * @param {string} headerValue
 * @param {object} entry
 */
function addContentType(headerValue, entry) {
  var value = headerValue.split(";").shift().toLowerCase();
  entry.contentType = value;

  switch (value) {
    case "text/html":
      entry.type = "html";
      entry.isHTML = true;
      break;

    case "text/xml":
      entry.type = "xml";
      entry.isXML = true;
      break;

    case "text/css":
      entry.type = "css";
      entry.isCSS = true;
      break;

    case "application/x-javascript":
    case "application/javascript":
    case "text/javascript":
      entry.type = "js";
      entry.isJS = true;
      break;

    case "application/json":
      entry.type = "json";
      entry.isJSON = true;
      break;

    case "image/png":
    case "image/jpeg":
    case "image/gif":
    case "image/svg+xml":
    case "image/webp":
      entry.type = "image";
      entry.isImage = true;

      if (value === "image/svg+xml") {
        entry.isSVG = true;
      }
      break;

    case "video/webm":
      entry.type = "video";
      entry.isVideo = true;
      break;

    // @see http://stackoverflow.com/questions/2871655/proper-mime-type-for-fonts
    case "application/font-wof":
    case "application/font-woff":
    case "application/font-woff2":
    case "application/vnd.ms-fontobject":
    case "application/x-font-opentype":
    case "application/x-font-truetype":
    case "application/x-font-ttf":
    case "application/x-font-woff":
    case "font/opentype":
    case "font/ttf":
    case "font/woff":
    case "font/woff2":
      entry.type = "webfont";
      entry.isWebFont = true;

      if (/ttf|truetype$/.test(value)) {
        entry.isTTF = true;
      }<commit_msg>Detect MP4 videos<commit_after>/**
 * Simple HTTP requests monitor and analyzer
 */
"use strict";

const assert = require("assert"),
  debug = require("debug")("phantomas:modules:requestsMonitor");

/**
 * Given key-value set of HTTP headers returns the set with lowercased header names
 *
 * @param {object} headers
 * @returns {object}
 */
function lowerCaseHeaders(headers) {
  var res = {};

  Object.keys(headers).forEach((headerName) => {
    res[headerName.toLowerCase()] = headers[headerName];
  });

  return res;
}

// parse given URL to get protocol and domain
function parseEntryUrl(entry) {
  var parsed;

  // asset type
  entry.type = "other";

  if (entry.url.indexOf("data:") === 0) {
    // base64 encoded data
    entry.domain = false;
    entry.protocol = false;
    entry.isBase64 = true;
  } else if (entry.url.indexOf("blob:") === 0) {
    // blob image or video
    entry.domain = false;
    entry.protocol = false;
    entry.isBlob = true;
  } else {
    parsed = new URL(entry.url) || {};

    entry.protocol = parsed.protocol.replace(":", ""); // e.g. "http:"
    entry.domain = parsed.hostname;
    entry.query = parsed.search.substring(1);

    if (entry.protocol === "https") {
      entry.isSSL = true;
    }
  }

  return entry;
}

/**
 * Detect response content type using "Content-Type header value"
 *
 * @param {string} headerValue
 * @param {object} entry
 */
function addContentType(headerValue, entry) {
  var value = headerValue.split(";").shift().toLowerCase();
  entry.contentType = value;

  switch (value) {
    case "text/html":
      entry.type = "html";
      entry.isHTML = true;
      break;

    case "text/xml":
      entry.type = "xml";
      entry.isXML = true;
      break;

    case "text/css":
      entry.type = "css";
      entry.isCSS = true;
      break;

    case "application/x-javascript":
    case "application/javascript":
    case "text/javascript":
      entry.type = "js";
      entry.isJS = true;
      break;

    case "application/json":
      entry.type = "json";
      entry.isJSON = true;
      break;

    case "image/png":
    case "image/jpeg":
    case "image/gif":
    case "image/svg+xml":
    case "image/webp":
      entry.type = "image";
      entry.isImage = true;

      if (value === "image/svg+xml") {
        entry.isSVG = true;
      }
      break;

    case "video/webm":
    case "video/mp4":
      entry.type = "video";
      entry.isVideo = true;
      break;

    // @see http://stackoverflow.com/questions/2871655/proper-mime-type-for-fonts
    case "application/font-wof":
    case "application/font-woff":
    case "application/font-woff2":
    case "application/vnd.ms-fontobject":
    case "application/x-font-opentype":
    case "application/x-font-truetype":
    case "application/x-font-ttf":
    case "application/x-font-woff":
    case "font/opentype":
    case "font/ttf":
    case "font/woff":
    case "font/woff2":
      entry.type = "webfont";
      entry.isWebFont = true;

      if (/ttf|truetype$/.test(value)) {
        entry.isTTF = true;
      }
{'fraction_non_alphanumeric': 0.15392189038398424, 'fraction_numerical': 0.005579258286839514, 'mean_word_length': 2.585294117647059, 'pattern_counts': {'":': 53, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4433198', 'n_tokens_mistral': 2036, 'n_tokens_neox': 1973, 'n_words': 456}