<commit_before>
/**
 * Convert the IANA definitions from CSV to local.
 */

var co = require('co')
var fs = require('fs')
var getRawBody = require('raw-body')
var cogent = require('cogent')
var parser = require('csv-parse')
var toArray = require('stream-to-array')

var leadingSpacesRegExp = /^\s+/
var listColonRegExp = /:(?:\s|$)/m
var mimeTypeLineRegExp = /^(?:\s*|[^:\s-]*\s+)(?:MIME type(?: name)?|MIME media type(?: name)?|Media type(?: name)?|Type name)\s?:\s+(.*)$/im
var mimeSubtypeLineRegExp = /^[^:\s-]*\s*(?:MIME |Media )?subtype(?: name)?\s?:\s+(?:[a-z]+ Tree (?:\- ?)?|(?:[a-z]+ )+\- )?([^\(\[\r\n]*).*$/im
var mimeSubtypesLineRegExp = /^[^:\s-]*\s*(?:MIME |Media )?subtype(?: names)?\s?:\s+(?:[a-z]+ Tree (?:\- ?)?)?(.*)$/im
var symbolRegExp = /[\._-]/g
var trimQuotesRegExp = /^"|"$/gm

co(function* () {
  var gens = yield [
    get('application'),
    get('audio'),
    get('image'),
    get('message'),
    get('model'),
    get('multipart'),
    get('text'),
  ]

  // flatten generators
  gens = gens.reduce(concat, [])

  // get results in groups
  var results = []
  while (gens.length !== 0) {
    results.push(yield gens.splice(0, 10))
  }

  // flatten results
  results = results.reduce(concat, [])

  var json = {}
  results.forEach(function (result) {
    var mime = result.mime

    if (mime in json && result.template !== json[mime].template) {
      throw new Error('duplicate entry for ' + mime)
    }

    delete result.mime
    json[mime] = result
  })

  fs.writeFileSync('src/iana.json', JSON.stringify(json))
})()

function addTemplateData(data) {
  if (!data.template) {
    return data
  }

  return function* get() {
    var res = yield* cogent('http://www.iana.org/assignments/media-types/' + data.template)
    var ref = data.type + '/' + data.name.replace(/ .*/, '')

    if (res.statusCode === 404 && data.template !== ref) {
      console.log('template ' + data.template + ' not found')
      data.template = ref
      res = yield* cogent('http://www.iana.org/assignments/media-types/' + ref)
    }

    if (res.statusCode === 404) {
      console.log('template ' + data.template + ' not found')
      return data
    }

    if (res.statusCode !== 200)
      throw new Error('got status code ' + res.statusCode + ' from template ' + data.template)

    var body = yield getTemplateBody(res)
    var mime = extractTemplateMime(body, ref)

    // use extracted mime if it's almost the same
    if (mime && mime.replace(symbolRegExp, '-') === data.mime.replace(symbolRegExp, '-')) {
      data.mime = mime
    }

    return data
  }
}

function extractTemplateMime(body, ref) {
  var type = mimeTypeLineRegExp.exec(body)
  var subtype = mimeSubtypeLineRegExp.exec(body)

  if (!subtype && (subtype = mimeSubtypesLineRegExp.exec(body)) && !/^[A-Za-z0-9]+$/.test(subtype[1])) {
    return
  }

  if (!type || !subtype) {
    //console.dir([ref, type&&type[1], subtype&&subtype[1]])
    return
  }

  type = type[1].trim().replace(trimQuotesRegExp, '')
  subtype = subtype[1].trim().replace(trimQuotesRegExp, '')

  if (!subtype) {
    return
  }

  if (subtype.substr(0, type.length + 1) === type + '/') {
    // strip type from subtype
    subtype = subtype.substr(type.length + 1)
  }

  return (type + '/' + subtype).toLowerCase()
}

function* get(type) {
  var res = yield* cogent('http://www.iana.org/assignments/media-types/' + encodeURIComponent(type) + '.csv')
  if (res.statusCode !== 200)
    throw new Error('got status code ' + res.statusCode + ' from ' + type)

  var mimes = yield toArray(res.pipe(parser()))
  var headers = mimes.shift().map(normalizeHeader)
  var reduceRows = generateRowMapper(headers)<commit_msg>build: remove debugging from types script<commit_after>
/**
 * Convert the IANA definitions from CSV to local.
 */

var co = require('co')
var fs = require('fs')
var getRawBody = require('raw-body')
var cogent = require('cogent')
var parser = require('csv-parse')
var toArray = require('stream-to-array')

var leadingSpacesRegExp = /^\s+/
var listColonRegExp = /:(?:\s|$)/m
var mimeTypeLineRegExp = /^(?:\s*|[^:\s-]*\s+)(?:MIME type(?: name)?|MIME media type(?: name)?|Media type(?: name)?|Type name)\s?:\s+(.*)$/im
var mimeSubtypeLineRegExp = /^[^:\s-]*\s*(?:MIME |Media )?subtype(?: name)?\s?:\s+(?:[a-z]+ Tree (?:\- ?)?|(?:[a-z]+ )+\- )?([^\(\[\r\n]*).*$/im
var mimeSubtypesLineRegExp = /^[^:\s-]*\s*(?:MIME |Media )?subtype(?: names)?\s?:\s+(?:[a-z]+ Tree (?:\- ?)?)?(.*)$/im
var symbolRegExp = /[\._-]/g
var trimQuotesRegExp = /^"|"$/gm

co(function* () {
  var gens = yield [
    get('application'),
    get('audio'),
    get('image'),
    get('message'),
    get('model'),
    get('multipart'),
    get('text'),
  ]

  // flatten generators
  gens = gens.reduce(concat, [])

  // get results in groups
  var results = []
  while (gens.length !== 0) {
    results.push(yield gens.splice(0, 10))
  }

  // flatten results
  results = results.reduce(concat, [])

  var json = {}
  results.forEach(function (result) {
    var mime = result.mime

    if (mime in json && result.template !== json[mime].template) {
      throw new Error('duplicate entry for ' + mime)
    }

    delete result.mime
    json[mime] = result
  })

  fs.writeFileSync('src/iana.json', JSON.stringify(json))
})()

function addTemplateData(data) {
  if (!data.template) {
    return data
  }

  return function* get() {
    var res = yield* cogent('http://www.iana.org/assignments/media-types/' + data.template)
    var ref = data.type + '/' + data.name.replace(/ .*/, '')

    if (res.statusCode === 404 && data.template !== ref) {
      console.log('template ' + data.template + ' not found')
      data.template = ref
      res = yield* cogent('http://www.iana.org/assignments/media-types/' + ref)
    }

    if (res.statusCode === 404) {
      console.log('template ' + data.template + ' not found')
      return data
    }

    if (res.statusCode !== 200)
      throw new Error('got status code ' + res.statusCode + ' from template ' + data.template)

    var body = yield getTemplateBody(res)
    var mime = extractTemplateMime(body)

    // use extracted mime if it's almost the same
    if (mime && mime.replace(symbolRegExp, '-') === data.mime.replace(symbolRegExp, '-')) {
      data.mime = mime
    }

    return data
  }
}

function extractTemplateMime(body) {
  var type = mimeTypeLineRegExp.exec(body)
  var subtype = mimeSubtypeLineRegExp.exec(body)

  if (!subtype && (subtype = mimeSubtypesLineRegExp.exec(body)) && !/^[A-Za-z0-9]+$/.test(subtype[1])) {
    return
  }

  if (!type || !subtype) {
    return
  }

  type = type[1].trim().replace(trimQuotesRegExp, '')
  subtype = subtype[1].trim().replace(trimQuotesRegExp, '')

  if (!subtype) {
    return
  }

  if (subtype.substr(0, type.length + 1) === type + '/') {
    // strip type from subtype
    subtype = subtype.substr(type.length + 1)
  }

  return (type + '/' + subtype).toLowerCase()
}

function* get(type) {
  var res = yield* cogent('http://www.iana.org/assignments/media-types/' + encodeURIComponent(type) + '.csv')
  if (res.statusCode !== 200)
    throw new Error('got status code ' + res.statusCode + ' from ' + type)

  var mimes = yield toArray(res.pipe(parser()))
  var headers = mimes.shift().map(normalizeHeader)
  var reduceRows = generateRowMapper(headers)
{'fraction_non_alphanumeric': 0.19997260649226134, 'fraction_numerical': 0.006848376934666484, 'mean_word_length': 3.250291036088475, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5476306', 'n_tokens_mistral': 2673, 'n_tokens_neox': 2546, 'n_words': 547}