<commit_before>(function() {
  var shared = {},
      HTML_BUNDLE_MIME_TYPE = "application/x-html-bundle+charlotte";

  if (typeof module !== 'undefined' && module.exports) {
    module.exports = shared;
  } else {
    this.charlotte = this.charlotte || {};
    this.charlotte.shared = shared;
  }
  
  function parseUrl(url) {
    var parsed;
    if ('undefined' !== typeof require && require('url')) {
      parsed = require('url').parse(url);
    } else if ('undefined' !== typeof window) {
      parsed = document.createElement('a');
      parsed.href = url;
      parsed.path = parsed.pathname;
    } else {
      throw new Error("Can't parse url, don't know where the hell I am.");
    }
    return parsed;
  };

  function removeMultislashes(s) {
    return s.replace(/([^:])\/{2,}/g, '$1/').
             replace(/^\/{2,}/, '/');
  }

  function removeUrlMultislashes(url) {
    if (url.match(/^(file|http|https)\:/)) {
      var parsedUrl = parseUrl(url);
      parsedUrl.pathname = removeMultislashes(unescape(parsedUrl.pathname));
      return parsedUrl.href;
    } else {
      return removeMultislashes(url);
    }
  }

  shared.HTML_BUNDLE_MIME_TYPE = HTML_BUNDLE_MIME_TYPE;
  
  shared.parseUrl = parseUrl;
  shared.removeUrlMultislashes = removeUrlMultislashes;
  
  shared.isBlank = function(varName, globally) {
    with(this) {
      var theVarIsBlank = eval("(" + 
                      '"undefined" === typeof ' + varName +  
                      '|| ' + varName + ' === null || ' +
                      varName + ' === "" )'),
          globalObj;
      if ('undefined' !== typeof global) globalObj = global;
      else if ('undefined' !== typeof window) globalObj = window;
      else throw new Error("Unknown global object.");
      if (globally) {
        return theVarIsBlank;
      } else {
        return theVarIsBlank || eval('(' + varName + ')') === globalObj[varName];
      }
    }<commit_msg>fix bug with removeUrlMultislashes() when executing in node<commit_after>(function() {
  var shared = {},
      HTML_BUNDLE_MIME_TYPE = "application/x-html-bundle+charlotte",
      urlModule;

  if (typeof require !==  'undefined' && require('url')) {
    urlModule = require('url');
  }

  if (typeof module !== 'undefined' && module.exports) {
    module.exports = shared;
  } else {
    this.charlotte = this.charlotte || {};
    this.charlotte.shared = shared;
  }
  
  function parseUrl(url) {
    var parsed;
    if (urlModule) {
      parsed = urlModule.parse(url);
    } else if ('undefined' !== typeof window) {
      parsed = document.createElement('a');
      parsed.href = url;
      parsed.path = parsed.pathname;
    } else {
      throw new Error("Can't parse url, don't know where the hell I am.");
    }
    return parsed;
  };

  function removeMultislashes(s) {
    return s.replace(/([^:])\/{2,}/g, '$1/').
             replace(/^\/{2,}/, '/');
  }

  function removeUrlMultislashes(url) {
    if (url.match(/^(file|http|https)\:/)) {
      var parsedUrl = parseUrl(url);
      parsedUrl.pathname = removeMultislashes(unescape(parsedUrl.pathname));
      if (urlModule) {
        return urlModule.format(parsedUrl);
      } else {
        return parsedUrl.href;
      }
    } else {
      return removeMultislashes(url);
    }
  }

  shared.HTML_BUNDLE_MIME_TYPE = HTML_BUNDLE_MIME_TYPE;
  
  shared.parseUrl = parseUrl;
  shared.removeUrlMultislashes = removeUrlMultislashes;
  
  shared.isBlank = function(varName, globally) {
    with(this) {
      var theVarIsBlank = eval("(" + 
                      '"undefined" === typeof ' + varName +  
                      '|| ' + varName + ' === null || ' +
                      varName + ' === "" )'),
          globalObj;
      if ('undefined' !== typeof global) globalObj = global;
      else if ('undefined' !== typeof window) globalObj = window;
      else throw new Error("Unknown global object.");
      if (globally) {
        return theVarIsBlank;
      } else {
        return theVarIsBlank || eval('(' + varName + ')') === globalObj[varName];
      }
    }
{'fraction_non_alphanumeric': 0.15341049002708693, 'fraction_numerical': 0.0014774686037921695, 'mean_word_length': 2.454081632653061, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '860577', 'n_tokens_mistral': 1280, 'n_tokens_neox': 1274, 'n_words': 263}