<commit_before>var _ = require('underscore');
var semver = require('semver');
var os = require('os');
var url = require('url');

var fiberHelpers = require('./fiber-helpers.js');
var archinfo = require('./archinfo.js');
var buildmessage = require('./buildmessage.js');
var files = require('../fs/files.js');
var packageVersionParser = require('../packaging/package-version-parser.js');

var utils = exports;

// Parses <protocol>://<host>:<port> into an object { protocol: *, host:
// *, port: * }. The input can also be of the form <host>:<port> or just
// <port>. We're not simply using 'url.parse' because we want '3000' to
// parse as {host: undefined, protocol: undefined, port: '3000'}, whereas
// 'url.parse' would give us {protocol:' 3000', host: undefined, port:
// undefined} or something like that.
//
// 'defaults' is an optional object with 'host', 'port', and 'protocol' keys.
exports.parseUrl = function (str, defaults) {
  // XXX factor this out into a {type: host/port}?

  defaults = defaults || {};
  var defaultHost = defaults.host || undefined;
  var defaultPort = defaults.port || undefined;
  var defaultProtocol = defaults.protocol || undefined;

  if (str.match(/^[0-9]+$/)) { // just a port
    return {
      port: str,
      host: defaultHost,
      protocol: defaultProtocol };
  }

  var hasScheme = exports.hasScheme(str);
  if (! hasScheme) {
    str = "http://" + str;
  }

  var parsed = url.parse(str);
  if (! parsed.protocol.match(/\/\/$/)) {
    // For easy concatenation, add double slashes to protocols.
    parsed.protocol = parsed.protocol + "//";
  }
  return {
    protocol: hasScheme ? parsed.protocol : defaultProtocol,
    host: parsed.hostname || defaultHost,
    port: parsed.port || defaultPort
  };
};

// 'url' is an object with 'host', 'port', and 'protocol' keys, such as
// the return value of parseUrl.
exports.formatUrl = function (url) {
  let string = url.protocol + url.host;
  if (url.port) {
    string += `:${url.port}`;
  }
  return string;
}

exports.ipAddress = function () {
  let defaultRoute;
  // netroute is not available on Windows
  if (process.platform !== "win32") {
    const netroute = require('netroute');
    const info = netroute.getInfo();
    defaultRoute = _.findWhere(info.IPv4 || [], { destination: "0.0.0.0" });
  }

  const interfaces = os.networkInterfaces();

  if (defaultRoute) {
    // If we know the default route, find the IPv4 address associated
    // with that interface
    const iface = defaultRoute["interface"];
    const addressEntry = _.findWhere(interfaces[iface], { family: "IPv4" });

    if (!addressEntry) {
      throw new Error(
`Network interface '${iface}', which seems to be the default route,
not found in interface list, or does not have an IPv4 address.`);
    }

    return addressEntry.address;
  } else {
    // If we don't know the default route, we'll lookup all non-internal
    // IPv4 addresses and hope to find only one
    let addressEntries = _.chain(interfaces).values().flatten().
      where({ family: "IPv4", internal: false }).value();

    if (addressEntries.length == 0) {
      throw new Error(
`Could not find a network interface with a non-internal IPv4 address.`);
    } else if (addressEntries.length > 1) {
      throw new Error(
`Found multiple network interfaces with non-internal IPv4 addresses:
${addressEntries.map(entry => entry.address).join(', ')}`);
    } else {
      return addressEntries[0].address;
    }
  }
};

exports.hasScheme = function (str) {
  return !! str.match(/^[A-Za-z][A-Za-z0-9+-\.]*\:\/\//);
};


exports.hasScheme = function (str) {
  return !! str.match(/^[A-Za-z][A-Za-z0-9+-\.]*\:\/\//);
};<commit_msg>Remove duplicate exports.hasScheme<commit_after>var _ = require('underscore');
var semver = require('semver');
var os = require('os');
var url = require('url');

var fiberHelpers = require('./fiber-helpers.js');
var archinfo = require('./archinfo.js');
var buildmessage = require('./buildmessage.js');
var files = require('../fs/files.js');
var packageVersionParser = require('../packaging/package-version-parser.js');

var utils = exports;

// Parses <protocol>://<host>:<port> into an object { protocol: *, host:
// *, port: * }. The input can also be of the form <host>:<port> or just
// <port>. We're not simply using 'url.parse' because we want '3000' to
// parse as {host: undefined, protocol: undefined, port: '3000'}, whereas
// 'url.parse' would give us {protocol:' 3000', host: undefined, port:
// undefined} or something like that.
//
// 'defaults' is an optional object with 'host', 'port', and 'protocol' keys.
exports.parseUrl = function (str, defaults) {
  // XXX factor this out into a {type: host/port}?

  defaults = defaults || {};
  var defaultHost = defaults.host || undefined;
  var defaultPort = defaults.port || undefined;
  var defaultProtocol = defaults.protocol || undefined;

  if (str.match(/^[0-9]+$/)) { // just a port
    return {
      port: str,
      host: defaultHost,
      protocol: defaultProtocol };
  }

  var hasScheme = exports.hasScheme(str);
  if (! hasScheme) {
    str = "http://" + str;
  }

  var parsed = url.parse(str);
  if (! parsed.protocol.match(/\/\/$/)) {
    // For easy concatenation, add double slashes to protocols.
    parsed.protocol = parsed.protocol + "//";
  }
  return {
    protocol: hasScheme ? parsed.protocol : defaultProtocol,
    host: parsed.hostname || defaultHost,
    port: parsed.port || defaultPort
  };
};

// 'url' is an object with 'host', 'port', and 'protocol' keys, such as
// the return value of parseUrl.
exports.formatUrl = function (url) {
  let string = url.protocol + url.host;
  if (url.port) {
    string += `:${url.port}`;
  }
  return string;
}

exports.ipAddress = function () {
  let defaultRoute;
  // netroute is not available on Windows
  if (process.platform !== "win32") {
    const netroute = require('netroute');
    const info = netroute.getInfo();
    defaultRoute = _.findWhere(info.IPv4 || [], { destination: "0.0.0.0" });
  }

  const interfaces = os.networkInterfaces();

  if (defaultRoute) {
    // If we know the default route, find the IPv4 address associated
    // with that interface
    const iface = defaultRoute["interface"];
    const addressEntry = _.findWhere(interfaces[iface], { family: "IPv4" });

    if (!addressEntry) {
      throw new Error(
`Network interface '${iface}', which seems to be the default route,
not found in interface list, or does not have an IPv4 address.`);
    }

    return addressEntry.address;
  } else {
    // If we don't know the default route, we'll lookup all non-internal
    // IPv4 addresses and hope to find only one
    let addressEntries = _.chain(interfaces).values().flatten().
      where({ family: "IPv4", internal: false }).value();

    if (addressEntries.length == 0) {
      throw new Error(
`Could not find a network interface with a non-internal IPv4 address.`);
    } else if (addressEntries.length > 1) {
      throw new Error(
`Found multiple network interfaces with non-internal IPv4 addresses:
${addressEntries.map(entry => entry.address).join(', ')}`);
    } else {
      return addressEntries[0].address;
    }
  }
};

exports.hasScheme = function (str) {
  return !! str.match(/^[A-Za-z][A-Za-z0-9+-\.]*\:\/\//);
};
{'fraction_non_alphanumeric': 0.15940035758492643, 'fraction_numerical': 0.009352221152523724, 'mean_word_length': 3.318289786223278, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7351384', 'n_tokens_mistral': 2264, 'n_tokens_neox': 2198, 'n_words': 697}