<commit_before>{
  "english": {
    "code": "en-US",
    "system": "urn:ietf:bcp:47",
    "display": "English"
  },
  "spanish": {
    "code": "es",
    "system": "urn:ietf:bcp:47",
    "display": "Spanish"
  },
  "portuguese": {
    "code": "pt",
    "system": "urn:ietf:bcp:47",
    "display": "Portuguese"
  },
  "chinese": {
    "code": "zh",
    "system": "urn:ietf:bcp:47",
    "display": "Chinese"
  },
  "japanese": {
    "code": "ja",
    "system": "urn:ietf:bcp:47",
    "display": "Japanese"
  },
  "korean": {
    "code": "ko",
    "system": "urn:ietf:bcp:47",
    "display": "Korean"
  },
  "french": {
    "code": "fr-FR",
    "system": "urn:ietf:bcp:47",
    "display": "French (France)"
  },
  "french_creole": {
    "code": "fr",
    "system": "urn:ietf:bcp:47",
    "display": "French"
  },
  "italian": {
    "code": "it",
    "system": "urn:ietf:bcp:47",
    "display": "Italian"
  },
  "russian": {
    "code": "ru-RU",
    "system": "urn:ietf:bcp:47",
    "display": "Russian (Russia)"
  },
  "vietnamese": {
    "code": "vi",
    "system": "http://id.loc.gov/vocabulary/iso639-1",
    "display": "Vietnamese"
  },
  "greek": {
    "code": "el",
    "system": "urn:ietf:bcp:47",
    "display": "Greek"
  },
  "arabic": {
    "code": "ar",
    "system": "http://id.loc.gov/vocabulary/iso639-1",
    "display": "Arabic"
  },
  "cambodian": {
    "code": "km",
    "system": "http://id.loc.gov/vocabulary/iso639-1",
    "display": "Khmer (aka Cambodian)"
  },
  "german": {
    "code": "de-DE",
    "system": "urn:ietf:bcp:47",
    "display": "German (Germany)"
  },
  "polish": {
    "code": "pl",
    "system": "urn:ietf:bcp:47",
    "display": "Polish"
  },
  "hawaiian": {
    "code": "haw",
    "system": "urn:ietf:bcp:47",
    "display": "Hawaiian"
  },
  "hindi": {
    "code": "hi",
    "system": "http://id.loc.gov/vocabulary/iso639-1",
    "display": "Hindi"
  },
  "undetermined": {
    "code": "und",
    "system": "http://id.loc.gov/vocabulary/iso639-1",
    "display": "Undetermined"
  }
}<commit_msg>Update language lookup to always use BCP: 47<commit_after>{
  "english": {
    "code": "en-US",
    "system": "urn:ietf:bcp:47",
    "display": "English"
  },
  "spanish": {
    "code": "es",
    "system": "urn:ietf:bcp:47",
    "display": "Spanish"
  },
  "portuguese": {
    "code": "pt",
    "system": "urn:ietf:bcp:47",
    "display": "Portuguese"
  },
  "chinese": {
    "code": "zh",
    "system": "urn:ietf:bcp:47",
    "display": "Chinese"
  },
  "japanese": {
    "code": "ja",
    "system": "urn:ietf:bcp:47",
    "display": "Japanese"
  },
  "korean": {
    "code": "ko",
    "system": "urn:ietf:bcp:47",
    "display": "Korean"
  },
  "french": {
    "code": "fr-FR",
    "system": "urn:ietf:bcp:47",
    "display": "French (France)"
  },
  "french_creole": {
    "code": "fr",
    "system": "urn:ietf:bcp:47",
    "display": "French"
  },
  "italian": {
    "code": "it",
    "system": "urn:ietf:bcp:47",
    "display": "Italian"
  },
  "russian": {
    "code": "ru-RU",
    "system": "urn:ietf:bcp:47",
    "display": "Russian (Russia)"
  },
  "vietnamese": {
    "code": "vi",
    "system": "urn:ietf:bcp:47",
    "display": "Vietnamese"
  },
  "greek": {
    "code": "el",
    "system": "urn:ietf:bcp:47",
    "display": "Greek"
  },
  "arabic": {
    "code": "ar",
    "system": "urn:ietf:bcp:47",
    "display": "Arabic"
  },
  "cambodian": {
    "code": "km",
    "system": "urn:ietf:bcp:47",
    "display": "Khmer (aka Cambodian)"
  },
  "german": {
    "code": "de-DE",
    "system": "urn:ietf:bcp:47",
    "display": "German (Germany)"
  },
  "polish": {
    "code": "pl",
    "system": "urn:ietf:bcp:47",
    "display": "Polish"
  },
  "hawaiian": {
    "code": "haw",
    "system": "urn:ietf:bcp:47",
    "display": "Hawaiian"
  },
  "hindi": {
    "code": "hi",
    "system": "urn:ietf:bcp:47",
    "display": "Hindi"
  },
  "undetermined": {
    "code": "und",
    "system": "urn:ietf:bcp:47",
    "display": "Undetermined"
  }
}
{'fraction_non_alphanumeric': 0.2622211080471296, 'fraction_numerical': 0.02206066683379293, 'mean_word_length': 2.9387956564659428, 'pattern_counts': {'":': 152, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7473295', 'n_tokens_mistral': 1846, 'n_tokens_neox': 1735, 'n_words': 285}