<commit_before>let http = require('follow-redirects').http;
let https = require('follow-redirects').https;
const {canonicalHost} = require('./host');

const balanceRegex = /([0-9]+) ?([a-zA-Z]*)/;

function joinPath() {
  const path = require('path');
  return path.join.apply(path.join, arguments);
}

function dirname() {
  const path = require('path');
  return path.dirname.apply(path.dirname, arguments);
}

function filesMatchingPattern(files) {
  const globule = require('globule');
  return globule.find(files, {nonull: true});
}

function fileMatchesPattern(patterns, intendedPath) {
  const globule = require('globule');
  return globule.isMatch(patterns, intendedPath);
}

function recursiveMerge(target, source) {
  const merge = require('merge');
  return merge.recursive(target, source);
}

function checkIsAvailable(url, callback) {
  http.get(url, function (_res) {
    callback(true);
  }).on('error', function (_res) {
    callback(false);
  });
}

function httpGetRequest(httpObj, url, callback) {
  httpObj.get(url, function (res) {
    let body = '';
    res.on('data', function (d) {
      body += d;
    });
    res.on('end', function () {
      callback(null, body);
    });
  }).on('error', function (err) {
    callback(err);
  });
}

function httpGet(url, callback) {
  httpGetRequest(http, url, callback);
}

function httpsGet(url, callback) {
  httpGetRequest(https, url, callback);
}

function httpGetJson(url, callback) {
  httpGetRequest(http, url, function (err, body) {
    try {
      let parsed = JSON.parse(body);
      return callback(err, parsed);
    } catch (e) {
      return callback(e);
    }
  });
}

function httpsGetJson(url, callback) {
  httpGetRequest(https, url, function (err, body) {
    try {
      let parsed = JSON.parse(body);
      return callback(err, parsed);
    } catch (e) {
      return callback(e);
    }
  });
}

function getJson(url, cb) {
  if (url.indexOf('https') === 0) {
    return httpsGetJson(url, cb);
  }
  httpGetJson(url, cb);
}

function pingEndpoint(host, port, type, protocol, origin, callback) {
  const options = {
    protocolVersion: 13,
    perMessageDeflate: true,
    origin: origin,
    host: host,
    port: port
  };
  if (type === 'ws') {
    options.headers = {
      'Sec-WebSocket-Version': 13,
      Connection: 'Upgrade',
      Upgrade: 'websocket',
      'Sec-WebSocket-Extensions': 'permessage-deflate; client_max_window_bits',
      Origin: origin
    };
  }
  let req;
  // remove trailing api key from infura, ie rinkeby.infura.io/nmY8WtT4QfEwz2S7wTbl
  if (options.host.indexOf('/') > -1) {
    options.host = options.host.split('/')[0];
  }
  if (protocol === 'https') {
    req = require('https').get(options);
  } else {
    req = require('http').get(options);
  }

  req.on('error', (err) => {
    callback(err);
  });

  req.on('response', (_response) => {
    callback();
  });

  req.on('upgrade', (_res, _socket, _head) => {
    callback();
  });
}

function runCmd(cmd, options) {
  const shelljs = require('shelljs');
  let result = shelljs.exec(cmd, options || {silent: true});
  if (result.code !== 0) {
    console.log("error doing.. " + cmd);
    console.log(result.output);
    if (result.stderr !== undefined) {
      console.log(result.stderr);
    }
    exit();
  }
}

function cd(folder) {
  const shelljs = require('shelljs');
  shelljs.cd(folder);
}

function sed(file, pattern, replace) {
  const shelljs = require('shelljs');
  shelljs.sed('-i', pattern, replace, file);
}

function exit(code) {
  process.exit(code);
}

function downloadFile(url, dest, cb) {
  const o_fs = require('fs-extra');
  var file = o_fs.createWriteStream(dest);
  (url.substring(0, 5) === 'https' ? https : http).get(url, function (response) {
    response.pipe(file);
    file.on('finish', function () {
      file.close(cb);
    });
  }).on('error', function (err) {
    o_fs.unlink(dest);
    if (cb) cb(err.message);
  });
}

function extractTar(filename, packageDirectory, cb) {
  const o_fs = require('fs-extra');
  const tar = require('tar');
  o_fs.createReadStream(filename).pipe(
    tar.x({
      strip: 1,
      C: packageDirectory
    }).on('end', function () {
      cb();
    })
  );<commit_msg>downloadFile should callback w/ err msg if response code is not 200<commit_after>let http = require('follow-redirects').http;
let https = require('follow-redirects').https;
const {canonicalHost} = require('./host');

const balanceRegex = /([0-9]+) ?([a-zA-Z]*)/;

function joinPath() {
  const path = require('path');
  return path.join.apply(path.join, arguments);
}

function dirname() {
  const path = require('path');
  return path.dirname.apply(path.dirname, arguments);
}

function filesMatchingPattern(files) {
  const globule = require('globule');
  return globule.find(files, {nonull: true});
}

function fileMatchesPattern(patterns, intendedPath) {
  const globule = require('globule');
  return globule.isMatch(patterns, intendedPath);
}

function recursiveMerge(target, source) {
  const merge = require('merge');
  return merge.recursive(target, source);
}

function checkIsAvailable(url, callback) {
  http.get(url, function (_res) {
    callback(true);
  }).on('error', function (_res) {
    callback(false);
  });
}

function httpGetRequest(httpObj, url, callback) {
  httpObj.get(url, function (res) {
    let body = '';
    res.on('data', function (d) {
      body += d;
    });
    res.on('end', function () {
      callback(null, body);
    });
  }).on('error', function (err) {
    callback(err);
  });
}

function httpGet(url, callback) {
  httpGetRequest(http, url, callback);
}

function httpsGet(url, callback) {
  httpGetRequest(https, url, callback);
}

function httpGetJson(url, callback) {
  httpGetRequest(http, url, function (err, body) {
    try {
      let parsed = JSON.parse(body);
      return callback(err, parsed);
    } catch (e) {
      return callback(e);
    }
  });
}

function httpsGetJson(url, callback) {
  httpGetRequest(https, url, function (err, body) {
    try {
      let parsed = JSON.parse(body);
      return callback(err, parsed);
    } catch (e) {
      return callback(e);
    }
  });
}

function getJson(url, cb) {
  if (url.indexOf('https') === 0) {
    return httpsGetJson(url, cb);
  }
  httpGetJson(url, cb);
}

function pingEndpoint(host, port, type, protocol, origin, callback) {
  const options = {
    protocolVersion: 13,
    perMessageDeflate: true,
    origin: origin,
    host: host,
    port: port
  };
  if (type === 'ws') {
    options.headers = {
      'Sec-WebSocket-Version': 13,
      Connection: 'Upgrade',
      Upgrade: 'websocket',
      'Sec-WebSocket-Extensions': 'permessage-deflate; client_max_window_bits',
      Origin: origin
    };
  }
  let req;
  // remove trailing api key from infura, ie rinkeby.infura.io/nmY8WtT4QfEwz2S7wTbl
  if (options.host.indexOf('/') > -1) {
    options.host = options.host.split('/')[0];
  }
  if (protocol === 'https') {
    req = require('https').get(options);
  } else {
    req = require('http').get(options);
  }

  req.on('error', (err) => {
    callback(err);
  });

  req.on('response', (_response) => {
    callback();
  });

  req.on('upgrade', (_res, _socket, _head) => {
    callback();
  });
}

function runCmd(cmd, options) {
  const shelljs = require('shelljs');
  let result = shelljs.exec(cmd, options || {silent: true});
  if (result.code !== 0) {
    console.log("error doing.. " + cmd);
    console.log(result.output);
    if (result.stderr !== undefined) {
      console.log(result.stderr);
    }
    exit();
  }
}

function cd(folder) {
  const shelljs = require('shelljs');
  shelljs.cd(folder);
}

function sed(file, pattern, replace) {
  const shelljs = require('shelljs');
  shelljs.sed('-i', pattern, replace, file);
}

function exit(code) {
  process.exit(code);
}

function downloadFile(url, dest, cb) {
  const o_fs = require('fs-extra');
  var file = o_fs.createWriteStream(dest);
  (url.substring(0, 5) === 'https' ? https : http).get(url, function (response) {
    if (response.statusCode !== 200) {
      if (cb) cb(`Download failed, response code ${response.statusCode}`);
    }
    response.pipe(file);
    file.on('finish', function () {
      file.close(cb);
    });
  }).on('error', function (err) {
    o_fs.unlink(dest);
    if (cb) cb(err.message);
  });
}

function extractTar(filename, packageDirectory, cb) {
  const o_fs = require('fs-extra');
  const tar = require('tar');
  o_fs.createReadStream(filename).pipe(
    tar.x({
      strip: 1,
      C: packageDirectory
    }).on('end', function () {
      cb();
    })
  );
{'fraction_non_alphanumeric': 0.17313953488372094, 'fraction_numerical': 0.004651162790697674, 'mean_word_length': 3.465732087227414, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2926127', 'n_tokens_mistral': 2937, 'n_tokens_neox': 2893, 'n_words': 639}