<commit_before>var semver = require("semver")
var depTypes = ["dependencies","devDependencies","optionalDependencies"]
var extractDescription = require("./extract_description")

module.exports = fixer = {
  fixRepositoryField: function(data) {
    if (data.repostories) {
      this.warn("'repositories' (plural) Not supported.\n" +
           "Please pick one as the 'repository' field");
      data.repository = data.repositories[0]
    }
    if (!data.repository) return;
    if (typeof data.repository === "string") {
      data.repository = {
        type: "git",
        url: data.repository
      }
    }
    var r = data.repository.url || ""
    // use the non-private urls
    r = r.replace(/^(https?|git):\/\/[^\@]+\@github.com/, 
      '$1://github.com')
    r = r.replace(/^https?:\/\/github.com/, 
      'git://github.com')
    if (r.match(/github.com\/[^\/]+\/[^\/]+\.git\.git$/)) {
      this.warn("Probably broken git url: " + r)
    }
  }

, fixFilesField: function(data) {
    var files = data.files
    if (files && !Array.isArray(files)) {
      this.warn("Invalid 'files' member")
      delete data.files
    }
  }

, fixBinField: function(data) {
    if (!data.bin) return;
    if (typeof data.bin === "string") {
      var b = {}
      b[data.name] = data.bin
      data.bin = b
    }
  }

, fixManField: function(data) {
    if (!data.man) return;
    if (typeof data.man === "string") {
      data.man = [ data.man ]
    }
  }
, fixBundleDependenciesField: function(data) {
    var bdd = "bundledDependencies"
    var bd = "bundleDependencies"
    if (data[bdd] && !data[bd]) {
      data[bd] = data[bdd]
      delete data[bdd]
    }
  }

, fixDependencies: function(data) {
    objectifyDeps(data, this.warn)
    addOptionalDepsToDeps(data, this.warn)
    this.fixBundleDependenciesField(data)
  }

, fixKeywordsField: function (data, warn) {
    if (typeof data.keywords === "string") {
      data.keywords = data.keywords.split(/,\s+/)
    }
  }

, fixVersionField: function(data) {
    if (!data.version) {
      data.version = ""
      return true
    }
    if (!semver.valid(data.version)) {
      throw new Error("invalid version: "+ data.version)
    }
    data.version = semver.clean(data.version)
    return true
  }

, fixPeople: function(data) {
    modifyPeople(data, unParsePerson)
    modifyPeople(data, parsePerson)  
  }  

, fixNameField: function(data) {
    if (!data.name) {
      data.name = ""
      return true
    }
    data.name = data.name.trim()
    ensureValidName(data.name)
  }
  

, fixDescriptionField: function (data) {
    if (data.description && typeof data.description !== 'string') {
      this.warn("'description' field should be a string")
      delete data.description
    }
    if (data.readme && !data.description)
      data.description = extractDescription(data.readme)
  }
  
, fixReadmeField: function (data) {
    if (!data.readme) data.readme = "ERROR: No README data found!"
  }
  
, fixBugsField: function(data) {
    var url
    if (!data.bugs && data.repository && data.repository.url) {
      url = data.repository.url
      // inference below was taken from code for "npm bugs" command
      if (url.match(/^(https?:\/\/|git(:\/\/|@))github.com/)) {
        url = url.replace(/^git(@|:\/\/)/, "https://")
                 .replace(/^https?:\/\/github.com:/, "https://github.com/")
                 .replace(/\.git$/, '')+"/issues"
        data.bugs = {url: url}
      }
    }
    else if(data.bugs && typeof data.bugs == "string") {
      data.bugs = {url: url}

,  fixHomepageField: function(data) {
    if(typeof data.homepage !== "string") {
      this.warn("homepage field must be a string. Deleted.")
      delete data.homepage
    }
  }
}

function ensureValidName (name) {
  if (name.charAt(0) === "." ||
      name.match(/[\/@\s\+%:]/) ||
      name !== encodeURIComponent(name) ||
      name.toLowerCase() === "node_modules" ||
      name.toLowerCase() === "favicon.ico") {
        throw new Error("Invalid name: " + JSON.stringify(name))
  }
}
<commit_msg>Ensure that if data.bugs exists, it is an object with (string) email, url fields, or both.<commit_after>var semver = require("semver")
var depTypes = ["dependencies","devDependencies","optionalDependencies"]
var extractDescription = require("./extract_description")

module.exports = fixer = {
  fixRepositoryField: function(data) {
    if (data.repostories) {
      this.warn("'repositories' (plural) Not supported.\n" +
           "Please pick one as the 'repository' field");
      data.repository = data.repositories[0]
    }
    if (!data.repository) return;
    if (typeof data.repository === "string") {
      data.repository = {
        type: "git",
        url: data.repository
      }
    }
    var r = data.repository.url || ""
    // use the non-private urls
    r = r.replace(/^(https?|git):\/\/[^\@]+\@github.com/, 
      '$1://github.com')
    r = r.replace(/^https?:\/\/github.com/, 
      'git://github.com')
    if (r.match(/github.com\/[^\/]+\/[^\/]+\.git\.git$/)) {
      this.warn("Probably broken git url: " + r)
    }
  }

, fixFilesField: function(data) {
    var files = data.files
    if (files && !Array.isArray(files)) {
      this.warn("Invalid 'files' member")
      delete data.files
    }
  }

, fixBinField: function(data) {
    if (!data.bin) return;
    if (typeof data.bin === "string") {
      var b = {}
      b[data.name] = data.bin
      data.bin = b
    }
  }

, fixManField: function(data) {
    if (!data.man) return;
    if (typeof data.man === "string") {
      data.man = [ data.man ]
    }
  }
, fixBundleDependenciesField: function(data) {
    var bdd = "bundledDependencies"
    var bd = "bundleDependencies"
    if (data[bdd] && !data[bd]) {
      data[bd] = data[bdd]
      delete data[bdd]
    }
  }

, fixDependencies: function(data) {
    objectifyDeps(data, this.warn)
    addOptionalDepsToDeps(data, this.warn)
    this.fixBundleDependenciesField(data)
  }

, fixKeywordsField: function (data, warn) {
    if (typeof data.keywords === "string") {
      data.keywords = data.keywords.split(/,\s+/)
    }
  }

, fixVersionField: function(data) {
    if (!data.version) {
      data.version = ""
      return true
    }
    if (!semver.valid(data.version)) {
      throw new Error("invalid version: "+ data.version)
    }
    data.version = semver.clean(data.version)
    return true
  }

, fixPeople: function(data) {
    modifyPeople(data, unParsePerson)
    modifyPeople(data, parsePerson)  
  }  

, fixNameField: function(data) {
    if (!data.name) {
      data.name = ""
      return true
    }
    data.name = data.name.trim()
    ensureValidName(data.name)
  }
  

, fixDescriptionField: function (data) {
    if (data.description && typeof data.description !== 'string') {
      this.warn("'description' field should be a string")
      delete data.description
    }
    if (data.readme && !data.description)
      data.description = extractDescription(data.readme)
  }
  
, fixReadmeField: function (data) {
    if (!data.readme) data.readme = "ERROR: No README data found!"
  }
  
, fixBugsField: function(data) {
    var url
    if (!data.bugs && data.repository && data.repository.url) {
      url = data.repository.url
      // inference below was taken from code for "npm bugs" command
      if (url.match(/^(https?:\/\/|git(:\/\/|@))github.com/)) {
        url = url.replace(/^git(@|:\/\/)/, "https://")
                 .replace(/^https?:\/\/github.com:/, "https://github.com/")
                 .replace(/\.git$/, '')+"/issues"
        data.bugs = {url: url}
      }
    }
    else if(data.bugs)
      if(typeof data.bugs == "string") {
        data.bugs = {url: data.bugs}
      }
      else {
        oldBugs = data.bugs
        data.bugs = {}
        if(oldBugs.url) {
          if(typeof(oldBugs.url) == "string")
            data.bugs.url = oldBugs.url
          else
            this.warn("bugs.url field must be a string. Deleted.")
        }
        if(oldBugs.email) {
          if(typeof(oldBugs.email) == "string")
            data.bugs.email = oldBugs.email
          else
            this.warn("bugs.email field must be a string. Deleted.")
        }
      }
      if(!data.bugs.email && !data.bugs.url) {
        delete data.bugs
        this.warn("Normalized value of bugs field is an empty object. Deleted.")
      }
  }

,  fixHomepageField: function(data) {
    if(typeof data.homepage !== "string") {
      this.warn("homepage field must be a string. Deleted.")
      delete data.homepage
    }
  }
}

function ensureValidName (name) {
  if (name.charAt(0) === "." ||
      name.match(/[\/@\s\+%:]/) ||
      name !== encodeURIComponent(name) ||
      name.toLowerCase() === "node_modules" ||
      name.toLowerCase() === "favicon.ico") {
        throw new Error("Invalid name: " + JSON.stringify(name))
  }
}

{'fraction_non_alphanumeric': 0.17000113032666442, 'fraction_numerical': 0.0006781959986436081, 'mean_word_length': 2.4083204930662556, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2519245', 'n_tokens_mistral': 2971, 'n_tokens_neox': 2912, 'n_words': 595}