<commit_before>//
// Basic schema loader that loads schemas over HTTP[S].
// Wrapper for http.get.
//
// If you were to write your own HTTP[S] loader, you’d probably want
// to use https://github.com/mikeal/request to do the heavy lifting.
//

var http = require('http')
  , https = require('https')
  , Errors = require('./errors.js')
  , url = require('url')
  ;

var MAX_DEPTH = 5;

function loader(ref, callback, _depth) {

  _depth = _depth || 0;
  if (_depth >= MAX_DEPTH) {
    var desc = 'could not GET URL: ' + ref + ' (too many redirects)';
    var err = new Errors.SchemaLoaderError(ref, desc);
    return callback(err);
  }

  var urlParts = url.parse(ref);

  var getter = http.get;
  if (urlParts.protocol === 'https:') { getter = https.get; }

  getter(ref, function(res) {
<commit_msg>workaround SSL error if CA certs not provided<commit_after>//
// Example schema loader that loads schemas over HTTP[S].
// Wrapper for http.get. For SSL connections, the CA certificate is
// not verified.
//
// If you were to write your own HTTP[S] loader, you’d probably want
// to use https://github.com/mikeal/request to do the heavy lifting.
//

var http = require('http')
  , https = require('https')
  , Errors = require('./errors.js')
  , url = require('url')
  ;

var MAX_DEPTH = 5;

function loader(ref, callback, _depth) {

  _depth = _depth || 0;
  if (_depth >= MAX_DEPTH) {
    var desc = 'could not GET URL: ' + ref + ' (too many redirects)';
    var err = new Errors.SchemaLoaderError(ref, desc);
    return callback(err);
  }

  var options = url.parse(ref);

  var getter = http.get;
  if (options.protocol === 'https:') {
    options.rejectUnauthorized = false;
    getter = https.get;
  }

  getter(options, function(res) {

{'fraction_non_alphanumeric': 0.14621559633027523, 'fraction_numerical': 0.0022935779816513763, 'mean_word_length': 3.2048192771084336, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4964813', 'n_tokens_mistral': 593, 'n_tokens_neox': 576, 'n_words': 186}