<commit_before>var cheerio = require('cheerio')
var map = require('map-limit')
var request = require('request')
var assign = require('object-assign')
var url = require('url')

var limit = 25

function scripts(body) {
  var $ = cheerio.load(body)
  var tags = []
  $('script').each(function(idx, el) {
    el = $(el)
    tags.push({
      type: el.attr('type'),
      src: el.attr('src'),
      body: el.html() || ''
    })
  })
  return tags
}

function fetch(uri, script, cb) {
  request.get({ uri: uri }, function(err, resp, body) {
    if (err) return cb(err)
    cb(null, assign({}, script, { body: body }))
  })
}

module.exports = function(opt, done) {
  if (typeof opt === 'string')
    opt = { uri: opt }
  else
    opt = opt||{}

  if (typeof done !== 'function')
    throw new TypeError('expected callback function')

  //first grab site body
  request.get(opt, function(err, resp, body) {
    if (err) return done(err)

    //parse script tags
    var tags = scripts(body)
    var base = opt.uri || opt.url

    //map script resources
    map(tags, limit, function(item, next) { 
      if (item.src) { //request and replace body
        var uri = url.resolve(base, item.src)
        fetch(uri, item, next)
      } else {
        next(null, item)
      }
    }, done)
  })
}<commit_msg>temp fix for base bath<commit_after>var cheerio = require('cheerio')
var map = require('map-limit')
var request = require('request')
var assign = require('object-assign')
var url = require('url')
var path = require('path')

var limit = 25

function scripts(body) {
  var $ = cheerio.load(body)
  var tags = []
  $('script').each(function(idx, el) {
    el = $(el)
    tags.push({
      type: el.attr('type'),
      src: el.attr('src'),
      body: el.html() || ''
    })
  })
  return tags
}

function fetch(base, script, cb) {
  request.get({ uri: script.src, baseUrl: base }, function(err, resp, body) {
    if (err) return cb(err)
    cb(null, assign({}, script, { body: body }))
  })
}

module.exports = function(opt, done) {
  if (typeof opt === 'string')
    opt = { uri: opt }
  else
    opt = opt||{}

  if (typeof done !== 'function')
    throw new TypeError('expected callback function')

  //first grab site body
  request.get(opt, function(err, resp, body) {
    if (err) return done(err)

    //parse script tags
    var tags = scripts(body)
    var baseUrl = url.parse(opt.uri || opt.url)
    var base = baseUrl.href
    if (path.extname(base)) {
      base = path.dirname(base)
    }

    //map script resources
    map(tags, limit, function(item, next) { 
      if (item.src) { //request and replace body
        fetch(base, item, next)
      } else {
        next(null, item)
      }
    }, done)
  })
}
{'fraction_non_alphanumeric': 0.16568483063328424, 'fraction_numerical': 0.0014727540500736377, 'mean_word_length': 2.742424242424242, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5020104', 'n_tokens_mistral': 921, 'n_tokens_neox': 905, 'n_words': 260}