<commit_before>
module.exports = readJson
readJson.processJson = processJson

var fs = require("fs")
  , semver = require("./semver")
  , log = require("../utils/log")

function readJson (jsonFile, tag, cb) {
  fs.readFile(jsonFile, processJson(tag, cb))
}
function processJson (tag, cb) {
  if (!cb) cb = tag, tag = null
  if (typeof cb !== "function") {
    var thing = cb, cb = null
    return P(null, thing)
  } else return P

  function P (er, thing) {
    if (er) {
      if (cb) return cb(er, thing)
      throw er
    }
    if (Object.prototype.toString.call(thing) === "[object Object]") {
      return processObject(tag, cb)(er, thing)
    } else {
      return processJsonString(tag, cb)(er, thing)
    }
  }
}
function processJsonString (tag, cb) { return function (er, jsonString) {
  jsonString += ""
  if (er) return cb(er, jsonString)
  var json
  try {
    json = JSON.parse(jsonString)
  } catch (ex) {
    var e = new Error(
      "Failed to parse json\n"+ex.message+"\n"+jsonString)
    if (cb) return cb(e)
    throw e
  }
  return processObject(tag, cb)(er, json)
}}
function processObject (tag, cb) { return function (er, json) {
  if (json.overlay) {
    ;["node", "npm"].forEach(function (k) {
      if (!json.overlay[k]) return undefined
      for (var i in json.overlay[k]) json[i] = json.overlay[k][i]
    })
  }

  // slashes would be a security risk.
  // anything else will just fail harmlessly.
  if (!json.name) {
    var e = new Error("No 'name' field found in package.json")
    if (cb) return cb(e)
    throw e
  }
  json.name = json.name.replace(/\//g, '-')
  if (tag && tag.indexOf(json.version) === 0) {
    tag = tag.substr(json.version.length)
  }
  if (tag) json.version += tag

  if (!(semver.valid(json.version))) {
    var e = new Error("Invalid version: "+json.version)
    if (cb) return cb(e)
    throw e
  }

  if (json.main) json.main = json.main.replace(/(\.js|\.node)$/, '')

  json._id = json.name+"-"+json.version
  json = testEngine(json)
  json = parsePeople(json)
  if (cb) cb(null,json)
  return json
}}

function testEngine (json) {
  if (!json.engines) json.engines = { "node" : "*" }

  var nodeVer = process.version.replace(/\+$/, '')
    , ok = false
  if (Array.isArray(json.engines)) {
    // Packages/1.0 commonjs style, with an array.
    // hack it to just hang a "node" member with the version range,
    // then do the npm-style check below.
    for (var i = 0, l = json.engines.length; i < l; i ++) {
      var e = json.engines[i].trim()
      if (e.substr(0, 4) === "node") {
        json.engines.node = e.substr(4)
        break
      }
    }
  }
  json._nodeSupported = semver.satisfies(nodeVer, json.engines.node || "undefined")
  return json
}

function parsePeople (json) {
  if (json.author) json.author = parsePerson(json.author)
  ;["maintainers", "contributors"].forEach(function (set) {
    if (Array.isArray(json[set])) json[set] = json[set].map(parsePerson)
  })
  return json
}
function parsePerson (person) {
  if (typeof person !== "string") return person
  var name = person.match(/^([^\(<]+)/)
    , url = person.match(/\(([^\)]+)\)/)
    , email = person.match(/<([^>]+)>/)
    , obj = { "name" : (name && name[0] || person).trim() }
  if (email) obj.email = email[1]
  if (url) obj.url = url[1]
  return obj
}
<commit_msg>eval the json as JavaScript if JSON.parse fails, so that comments work.<commit_after>
module.exports = readJson
readJson.processJson = processJson

var fs = require("fs")
  , semver = require("./semver")
  , log = require("../utils/log")

function readJson (jsonFile, tag, cb) {
  fs.readFile(jsonFile, processJson(tag, cb))
}
function processJson (tag, cb) {
  if (!cb) cb = tag, tag = null
  if (typeof cb !== "function") {
    var thing = cb, cb = null
    return P(null, thing)
  } else return P

  function P (er, thing) {
    if (er) {
      if (cb) return cb(er, thing)
      throw er
    }
    if (Object.prototype.toString.call(thing) === "[object Object]") {
      return processObject(tag, cb)(er, thing)
    } else {
      return processJsonString(tag, cb)(er, thing)
    }
  }
}
function processJsonString (tag, cb) { return function (er, jsonString) {
  jsonString += ""
  if (er) return cb(er, jsonString)
  var json
  try {
    json = JSON.parse(jsonString)
  } catch (ex2) { try {
    json = process.binding("evals").Script.runInNewContext("(\n"+jsonString+"\n)")
  } catch (ex) {
    var e = new Error(
      "Failed to parse json\n"+ex.message+"\n"+ex2.message+"\n"+jsonString)
    if (cb) return cb(e)
    throw e
  }}
  return processObject(tag, cb)(er, json)
}}
function processObject (tag, cb) { return function (er, json) {
  if (json.overlay) {
    ;["node", "npm"].forEach(function (k) {
      if (!json.overlay[k]) return undefined
      for (var i in json.overlay[k]) json[i] = json.overlay[k][i]
    })
  }

  // slashes would be a security risk.
  // anything else will just fail harmlessly.
  if (!json.name) {
    var e = new Error("No 'name' field found in package.json")
    if (cb) return cb(e)
    throw e
  }
  json.name = json.name.replace(/\//g, '-')
  if (tag && tag.indexOf(json.version) === 0) {
    tag = tag.substr(json.version.length)
  }
  if (tag) json.version += tag

  if (!(semver.valid(json.version))) {
    var e = new Error("Invalid version: "+json.version)
    if (cb) return cb(e)
    throw e
  }

  if (json.main) json.main = json.main.replace(/(\.js|\.node)$/, '')

  json._id = json.name+"-"+json.version
  json = testEngine(json)
  json = parsePeople(json)
  if (cb) cb(null,json)
  return json
}}

function testEngine (json) {
  if (!json.engines) json.engines = { "node" : "*" }

  var nodeVer = process.version.replace(/\+$/, '')
    , ok = false
  if (Array.isArray(json.engines)) {
    // Packages/1.0 commonjs style, with an array.
    // hack it to just hang a "node" member with the version range,
    // then do the npm-style check below.
    for (var i = 0, l = json.engines.length; i < l; i ++) {
      var e = json.engines[i].trim()
      if (e.substr(0, 4) === "node") {
        json.engines.node = e.substr(4)
        break
      }
    }
  }
  json._nodeSupported = semver.satisfies(nodeVer, json.engines.node || "undefined")
  return json
}

function parsePeople (json) {
  if (json.author) json.author = parsePerson(json.author)
  ;["maintainers", "contributors"].forEach(function (set) {
    if (Array.isArray(json[set])) json[set] = json[set].map(parsePerson)
  })
  return json
}
function parsePerson (person) {
  if (typeof person !== "string") return person
  var name = person.match(/^([^\(<]+)/)
    , url = person.match(/\(([^\)]+)\)/)
    , email = person.match(/<([^>]+)>/)
    , obj = { "name" : (name && name[0] || person).trim() }
  if (email) obj.email = email[1]
  if (url) obj.url = url[1]
  return obj
}

{'fraction_non_alphanumeric': 0.1763841973858129, 'fraction_numerical': 0.0032310177705977385, 'mean_word_length': 2.9273356401384083, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5179737', 'n_tokens_mistral': 2418, 'n_tokens_neox': 2391, 'n_words': 644}