<commit_before>var core                  = require("./core").dom.level2.core,
    events                = require("./core").dom.level2.events,
    applyDocumentFeatures = require('../browser/documentfeatures').applyDocumentFeatures,
    URL                   = require("url"),
    Path                  = require('path'),
    fs                    = require("fs"),
    http                  = require('http'),
    https                 = require('https');

// modify cloned instance for more info check: https://github.com/tmpvar/jsdom/issues/325
core = Object.create(core);

// Setup the javascript language processor
core.languageProcessors = {
  javascript : require("./languages/javascript").javascript
};

core.resourceLoader = {
  load: function(element, href, callback) {
    var ownerImplementation = element._ownerDocument.implementation;

    if (ownerImplementation.hasFeature('FetchExternalResources', element.tagName.toLowerCase())) {
      var full = this.resolve(element._ownerDocument, href);
      var url = URL.parse(full);
      if (url.hostname) {
        this.download(url, this.baseUrl(element._ownerDocument), this.enqueue(element, callback, full));
      }
      else {
        this.readFile(url.pathname, this.enqueue(element, callback, full));
      }
    }
  },
  enqueue: function(element, callback, filename) {
    var loader = this,
        doc    = element.nodeType === core.Node.DOCUMENT_NODE ?
                 element                :
                 element._ownerDocument;

    if (!doc._queue) {
      return function() {};
    }

    return doc._queue.push(function(err, data) {
      var ev = doc.createEvent('HTMLEvents');

      if (!err) {
        try {
          callback.call(element, data, filename || doc.URL);
          ev.initEvent('load', false, false);
        }
        catch(e) {
          err = e;
        }
      }

      if (err) {
        ev.initEvent('error', false, false);
        ev.error = err;
      }

      element.dispatchEvent(ev);
    });
  },

  baseUrl: function(document) {
    var baseElements = document.getElementsByTagName('base'),
        baseUrl      = document.URL;

    if (baseElements.length > 0) {
      baseUrl = baseElements.item(0).href;
    }

    return baseUrl;
  },
  resolve: function(document, href) {
    if (href.match(/^\w+:\/\//)) {
      return href;
    }

    var baseUrl = this.baseUrl(document);

    // See RFC 2396 section 3 for this weirdness. URLs without protocol
    // have their protocol default to the current one.
    // http://www.ietf.org/rfc/rfc2396.txt
    if (href.match(/^\/\//)) {
      return baseUrl ? baseUrl.match(/^(\w+:)\/\//)[1] + href : null;
    } else if (!href.match(/^\/[^\/]/)) {
      href = href.replace(/^\//, "");
    }

    return URL.resolve(baseUrl, href);
  },
  download: function(url, referrer, callback) {
    var path    = url.pathname + (url.search || ''),
        options = {'method': 'GET', 'host': url.hostname, 'path': path},
        request;
    if (url.protocol === 'https:') {
      options.port = url.port || 443;
      request = https.request(options);
    } else {
      options.port = url.port || 80;
      request = http.request(options);
    }

    // set header.
    if (referrer) {
        request.setHeader('Referer', referrer);
    }

    request.on('response', function (response) {
      var data = '';
      function success () {
        if ([301, 302, 303, 307].indexOf(response.statusCode) > -1) {
          var redirect = URL.resolve(url, response.headers["location"]);
          core.resourceLoader.download(URL.parse(redirect), referrer, callback);
        } else {
          callback(null, data);
        }
      }
      response.setEncoding('utf8');
      response.on('data', function (chunk) {
        data += chunk.toString();
      });
      response.on('end', function() {
        // According to node docs, 'close' can fire after 'end', but not
        // vice versa.  Remove 'close' listener so we don't call success twice.
        response.removeAllListeners('close');
        success();
      });
      response.on('close', function (err) {
        if (err) {
          callback(err);
        } else {
          success();
        }
      });
    });

    request.on('error', callback);
    request.end();
  },
  readFile: function(url, callback) {
    fs.readFile(url.replace(/^file:\/\//, ""), 'utf8', callback);
  }
};

function define(elementClass, def) {
  var tagName = def.tagName,
    tagNames = def.tagNames || (tagName? [tagName] : []),
    parentClass = def.parentClass || core.HTMLElement,
    attrs = def.attributes || [],
    proto = def.proto || {};

  var elem = core[elementClass] = function(document, name) {
    parentClass.call(this, document, name || tagName.toUpperCase());
    if (elem._init) {
      elem._init.call(this);
    }
  };
  elem._init = def.init;

  elem.prototype = proto;<commit_msg>Fix a test in jsdom/index on Windows due to file:///C:/ format.<commit_after>var core                  = require("./core").dom.level2.core,
    events                = require("./core").dom.level2.events,
    applyDocumentFeatures = require('../browser/documentfeatures').applyDocumentFeatures,
    URL                   = require("url"),
    Path                  = require('path'),
    fs                    = require("fs"),
    http                  = require('http'),
    https                 = require('https');

// modify cloned instance for more info check: https://github.com/tmpvar/jsdom/issues/325
core = Object.create(core);

// Setup the javascript language processor
core.languageProcessors = {
  javascript : require("./languages/javascript").javascript
};

core.resourceLoader = {
  load: function(element, href, callback) {
    var ownerImplementation = element._ownerDocument.implementation;

    if (ownerImplementation.hasFeature('FetchExternalResources', element.tagName.toLowerCase())) {
      var full = this.resolve(element._ownerDocument, href);
      var url = URL.parse(full);
      if (url.hostname) {
        this.download(url, this.baseUrl(element._ownerDocument), this.enqueue(element, callback, full));
      }
      else {
        this.readFile(url.pathname, this.enqueue(element, callback, full));
      }
    }
  },
  enqueue: function(element, callback, filename) {
    var loader = this,
        doc    = element.nodeType === core.Node.DOCUMENT_NODE ?
                 element                :
                 element._ownerDocument;

    if (!doc._queue) {
      return function() {};
    }

    return doc._queue.push(function(err, data) {
      var ev = doc.createEvent('HTMLEvents');

      if (!err) {
        try {
          callback.call(element, data, filename || doc.URL);
          ev.initEvent('load', false, false);
        }
        catch(e) {
          err = e;
        }
      }

      if (err) {
        ev.initEvent('error', false, false);
        ev.error = err;
      }

      element.dispatchEvent(ev);
    });
  },

  baseUrl: function(document) {
    var baseElements = document.getElementsByTagName('base'),
        baseUrl      = document.URL;

    if (baseElements.length > 0) {
      baseUrl = baseElements.item(0).href;
    }

    return baseUrl;
  },
  resolve: function(document, href) {
    if (href.match(/^\w+:\/\//)) {
      return href;
    }

    var baseUrl = this.baseUrl(document);

    // See RFC 2396 section 3 for this weirdness. URLs without protocol
    // have their protocol default to the current one.
    // http://www.ietf.org/rfc/rfc2396.txt
    if (href.match(/^\/\//)) {
      return baseUrl ? baseUrl.match(/^(\w+:)\/\//)[1] + href : null;
    } else if (!href.match(/^\/[^\/]/)) {
      href = href.replace(/^\//, "");
    }

    return URL.resolve(baseUrl, href);
  },
  download: function(url, referrer, callback) {
    var path    = url.pathname + (url.search || ''),
        options = {'method': 'GET', 'host': url.hostname, 'path': path},
        request;
    if (url.protocol === 'https:') {
      options.port = url.port || 443;
      request = https.request(options);
    } else {
      options.port = url.port || 80;
      request = http.request(options);
    }

    // set header.
    if (referrer) {
        request.setHeader('Referer', referrer);
    }

    request.on('response', function (response) {
      var data = '';
      function success () {
        if ([301, 302, 303, 307].indexOf(response.statusCode) > -1) {
          var redirect = URL.resolve(url, response.headers["location"]);
          core.resourceLoader.download(URL.parse(redirect), referrer, callback);
        } else {
          callback(null, data);
        }
      }
      response.setEncoding('utf8');
      response.on('data', function (chunk) {
        data += chunk.toString();
      });
      response.on('end', function() {
        // According to node docs, 'close' can fire after 'end', but not
        // vice versa.  Remove 'close' listener so we don't call success twice.
        response.removeAllListeners('close');
        success();
      });
      response.on('close', function (err) {
        if (err) {
          callback(err);
        } else {
          success();
        }
      });
    });

    request.on('error', callback);
    request.end();
  },
  readFile: function(url, callback) {
    fs.readFile(url.replace(/^file:\/\//, "").replace(/^\/([a-z]):\//i, '$1:/'), 'utf8', callback);
  }
};

function define(elementClass, def) {
  var tagName = def.tagName,
    tagNames = def.tagNames || (tagName? [tagName] : []),
    parentClass = def.parentClass || core.HTMLElement,
    attrs = def.attributes || [],
    proto = def.proto || {};

  var elem = core[elementClass] = function(document, name) {
    parentClass.call(this, document, name || tagName.toUpperCase());
    if (elem._init) {
      elem._init.call(this);
    }
  };
  elem._init = def.init;

  elem.prototype = proto;
{'fraction_non_alphanumeric': 0.15766039263307022, 'fraction_numerical': 0.007589556769884639, 'mean_word_length': 2.201490119857467, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7523770', 'n_tokens_mistral': 3012, 'n_tokens_neox': 2897, 'n_words': 627}