<commit_before># -*- coding: utf-8 -*-
"""Universal Language Codes

This library aims to provide an [en/de]coding utility for language codes.

To get a "universal" language code you create a LanguageCode object, giving it
the code and the standard it should use to look up that code.

    >>> lc = LanguageCode('en', 'iso-639-1')

Internally the code is stored in a custom standard designed specifically for
this purpose.  It doesn't have any use in the real world, so to get a useful
representation out you "encode" the code:

    >>> lc.encode('iso-639-2')
    'eng'

This is similar to Python's handling of Unicode and byte strings:

    >>> s = 'Hello, world'.decode('ascii')

    >>> s
    u'Hello, world'

    >>> s.encode('utf-8')
    'Hello, world'

"""

import copy


def _reverse_dict(d):
    return dict([(v, k) for k, v in d.items()])


# INTERNAL_NAMES stores the English and native names for the various languages.
#
# Code -> (English name, Native name)
#
# { 'ar': (u'Arabic', u'العربية'),
#   'el': (u'Greek', u'Ελληνικά'),
#   ... }
INTERNAL_NAMES = {}

# TO_INTERNAL is a dict of dicts.
#
# The first level is a mapping of standard names to their dicts.
#
# The second level (one per standard) is a mapping of that standard's language
# codes to the internal language codes.
#
# { 'iso-639-1': {
#      'ak': 'aka',
#      ...
#    },
#    ... }
TO_INTERNAL = {}

# FROM_INTERNAL is a dict of dicts.
#
# The first level is a mapping of standard names to their dicts.
#
# The second level (one per standard) is a mapping of internal language codes
# to that standard's language codes.
#
# { 'iso-639-1': {
#      'ak': 'aka',
#      ...
#    },
#    ... }
FROM_INTERNAL = {}


gettext_noop = lambda s: s

def add_standard(standard, mapping, base=None, exclude=None):
    """Add a new standard to the list of supported standards.

    `mapping` should be a dictionary mapping your custom standard's codes to the
    internal "universal" code used by this library.

    `base` is optional.  If given it will use the given standard as a base and
    copy all of its mappings before updating it with the ones you pass in
    through `mappings`.

    This can be useful for creating your own custom standard that's mostly like
    an existing one except for a few changes:

        >>> add_standard('my-standard', {'american': 'en'}, base='iso-639-1')

    This example creates a new custom standard, which is pretty much like
    ISO-639-1 but adds a code called 'american' that represents the English
    language.  Now you can do:

        >>> lc = LanguageCode('american', 'my-standard')
        >>> lc.encode('iso-639-2')
        'en'

    You can pass a list of codes to exclude from the base through the `exclude`
    parameter:

        >>> add_standard('my-standard', {'american': 'en'},
                         base='iso-639-1', exclude=('no', 'en'))

    """
    if base:
        m = copy.copy(TO_INTERNAL[base])
        m.update(mapping)

        if exclude:
            for c in exclude:
                del m[c]
    else:
        m = mapping

    TO_INTERNAL[standard] = m
    FROM_INTERNAL[standard] = _reverse_dict(m)

def _generate_initial_data():
    INTERNAL_NAMES.update({
        'aa': (gettext_noop(u'Afar'), u'Afar'),
        'ab': (gettext_noop(u'Abkhazian'), u'Abkhazian'),
        'ae': (gettext_noop(u'Avestan'), u'Avestan'),
        'af': (gettext_noop(u'Afrikaans'), u'Afrikaans'),
        'aka': (gettext_noop(u'Akan'), u'Akana'),
        'amh': (gettext_noop(u'Amharic'), u'Amharic'),
        'an': (gettext_noop(u'Aragonese'), u'Aragonés'),
        'ar': (gettext_noop(u'Arabic'), u'العربية'),
        'as': (gettext_noop(u'Assamese'), u'Assamese'),
        'ase': (gettext_noop(u'American Sign Language'), u'American Sign Language'),
        'ast': (gettext_noop(u'Asturian'), u'Asturianu'),
        'av': (gettext_noop(u'Avaric'), u'Авар'),
        'ay': (gettext_noop(u'Aymara'), u'Aymar'),
        'az': (gettext_noop(u'Azerbaijani'), u'Azərbaycan'),
        'ba': (gettext_noop(u'Bashkir'), u'Башҡорт'),
        'bam': (gettext_noop(u'Bambara'), u'Bamanankan'),
        'be': (gettext_noop(u'Belarusian'), u'Беларуская'),
        'ber': (gettext_noop(u'Berber'), u'Berber'),
        'bg': (gettext_noop(u'Bulgarian'), u'Български'),
        'bh': (gettext_noop(u'Bihari'), u'भोजपुर'),
        'bi': (gettext_noop(u'Bislama'), u'Bislama'),
        'bn': (gettext_noop(u'Bengali'), u'Bengali'),
        'bnt': (gettext_noop(u'Ibibio'), u'Ibibio'),
        'bo': (gettext_noop(u'Tibetan'), u'Bod skad'),
        'br': (gettext_noop(u'Breton'), u'Brezhoneg'),
        'bs': (gettext_noop(u'Bosnian'), u'Bosanski'),
        'ca': (gettext_noop(u'Catalan'), u'Català'),
        'ce': (gettext_noop(u'Chechen'), u'Chechen'),
        'ceb': (gettext_noop(u'Cebuano'), u'Cebuano'),
        'ch': (gettext_noop(u'Chamorro'), u'Chamoru'),
        'cho': (gettext_noop(u'Choctaw'), u'Choctaw'),
        'co': (gettext_noop(u'Corsican'), u'Corsu'),
        'cr': (gettext_noop(u'Cree'), u'Nehiyaw'),
        'cs': (gettext_noop(u'Czech'), u'Čeština'),
        'cu': (gettext_noop(u'Church Slavic'), u'Church Slavic'),
        'cv': (gettext_noop(u'Chuvash'), u'Chuvash'),
        'cy': (gettext_noop(u'Welsh'), u'Cymraeg'),
        'da': (gettext_noop(u'Danish'), u'Dansk'),
        'de': (gettext_noop(u'German'), u'Deutsch'),
        'dv': (gettext_noop(u'Divehi'), u'Divehi'),
        'dz': (gettext_noop(u'Dzongkha'), u'Dzongkha'),
        'ee': (gettext_noop(u'Ewe'), u'Ewe'),
        'efi': (gettext_noop(u'Efik'), u'Efik'),
        'el': (gettext_noop(u'Greek'), u'Ελληνικά'),
        'en': (gettext_noop(u'English'), u'English'),
        'en-gb': (gettext_noop(u'English, British'), u'English, British'),
        'eo': (gettext_noop(u'Esperanto'), u'Esperanto'),
        'es': (gettext_noop(u'Spanish'), u'Español'),
        'es-ar': (gettext_noop(u'Spanish, Argentinian'), u'Spanish, Argentinian'),
        'es-mx': (gettext_noop(u'Mexican Spanish'), u'Mexican Spanish'),
        'es-ni': (gettext_noop(u'Nicaraguan Spanish'), u'Nicaraguan Spanish'),
        'et': (gettext_noop(u'Estonian'), u'Eesti'),
        'eu': (gettext_noop(u'Basque'), u'Euskara'),
        'fa': (gettext_noop(u'Persian'), u'فارسی'),
        'ff': (gettext_noop(u'Fulah'), u'Fulah'),
        'fi': (gettext_noop(u'Finnish'), u'Suomi'),
        'fil': (gettext_noop(u'Filipino'), u'Filipino'),
        'fj': (gettext_noop(u'Fijian'), u'Na Vosa Vakaviti'),
        'fo': (gettext_noop(u'Faroese'), u'Føroyskt'),
        'fr': (gettext_noop(u'French'), u'Français'),
        'fr-ca': (gettext_noop(u'French, Canadian'), u'French, Canadian'),
        'ful': (gettext_noop(u'Fula'), u'Fula'),
        'fy-nl': (gettext_noop(u'Frisian'), u'Frysk'),<commit_msg>Fix a few dialect names.<commit_after># -*- coding: utf-8 -*-
"""Universal Language Codes

This library aims to provide an [en/de]coding utility for language codes.

To get a "universal" language code you create a LanguageCode object, giving it
the code and the standard it should use to look up that code.

    >>> lc = LanguageCode('en', 'iso-639-1')

Internally the code is stored in a custom standard designed specifically for
this purpose.  It doesn't have any use in the real world, so to get a useful
representation out you "encode" the code:

    >>> lc.encode('iso-639-2')
    'eng'

This is similar to Python's handling of Unicode and byte strings:

    >>> s = 'Hello, world'.decode('ascii')

    >>> s
    u'Hello, world'

    >>> s.encode('utf-8')
    'Hello, world'

"""

import copy


def _reverse_dict(d):
    return dict([(v, k) for k, v in d.items()])


# INTERNAL_NAMES stores the English and native names for the various languages.
#
# Code -> (English name, Native name)
#
# { 'ar': (u'Arabic', u'العربية'),
#   'el': (u'Greek', u'Ελληνικά'),
#   ... }
INTERNAL_NAMES = {}

# TO_INTERNAL is a dict of dicts.
#
# The first level is a mapping of standard names to their dicts.
#
# The second level (one per standard) is a mapping of that standard's language
# codes to the internal language codes.
#
# { 'iso-639-1': {
#      'ak': 'aka',
#      ...
#    },
#    ... }
TO_INTERNAL = {}

# FROM_INTERNAL is a dict of dicts.
#
# The first level is a mapping of standard names to their dicts.
#
# The second level (one per standard) is a mapping of internal language codes
# to that standard's language codes.
#
# { 'iso-639-1': {
#      'ak': 'aka',
#      ...
#    },
#    ... }
FROM_INTERNAL = {}


gettext_noop = lambda s: s

def add_standard(standard, mapping, base=None, exclude=None):
    """Add a new standard to the list of supported standards.

    `mapping` should be a dictionary mapping your custom standard's codes to the
    internal "universal" code used by this library.

    `base` is optional.  If given it will use the given standard as a base and
    copy all of its mappings before updating it with the ones you pass in
    through `mappings`.

    This can be useful for creating your own custom standard that's mostly like
    an existing one except for a few changes:

        >>> add_standard('my-standard', {'american': 'en'}, base='iso-639-1')

    This example creates a new custom standard, which is pretty much like
    ISO-639-1 but adds a code called 'american' that represents the English
    language.  Now you can do:

        >>> lc = LanguageCode('american', 'my-standard')
        >>> lc.encode('iso-639-2')
        'en'

    You can pass a list of codes to exclude from the base through the `exclude`
    parameter:

        >>> add_standard('my-standard', {'american': 'en'},
                         base='iso-639-1', exclude=('no', 'en'))

    """
    if base:
        m = copy.copy(TO_INTERNAL[base])
        m.update(mapping)

        if exclude:
            for c in exclude:
                del m[c]
    else:
        m = mapping

    TO_INTERNAL[standard] = m
    FROM_INTERNAL[standard] = _reverse_dict(m)

def _generate_initial_data():
    INTERNAL_NAMES.update({
        'aa': (gettext_noop(u'Afar'), u'Afar'),
        'ab': (gettext_noop(u'Abkhazian'), u'Abkhazian'),
        'ae': (gettext_noop(u'Avestan'), u'Avestan'),
        'af': (gettext_noop(u'Afrikaans'), u'Afrikaans'),
        'aka': (gettext_noop(u'Akan'), u'Akana'),
        'amh': (gettext_noop(u'Amharic'), u'Amharic'),
        'an': (gettext_noop(u'Aragonese'), u'Aragonés'),
        'ar': (gettext_noop(u'Arabic'), u'العربية'),
        'as': (gettext_noop(u'Assamese'), u'Assamese'),
        'ase': (gettext_noop(u'American Sign Language'), u'American Sign Language'),
        'ast': (gettext_noop(u'Asturian'), u'Asturianu'),
        'av': (gettext_noop(u'Avaric'), u'Авар'),
        'ay': (gettext_noop(u'Aymara'), u'Aymar'),
        'az': (gettext_noop(u'Azerbaijani'), u'Azərbaycan'),
        'ba': (gettext_noop(u'Bashkir'), u'Башҡорт'),
        'bam': (gettext_noop(u'Bambara'), u'Bamanankan'),
        'be': (gettext_noop(u'Belarusian'), u'Беларуская'),
        'ber': (gettext_noop(u'Berber'), u'Berber'),
        'bg': (gettext_noop(u'Bulgarian'), u'Български'),
        'bh': (gettext_noop(u'Bihari'), u'भोजपुर'),
        'bi': (gettext_noop(u'Bislama'), u'Bislama'),
        'bn': (gettext_noop(u'Bengali'), u'Bengali'),
        'bnt': (gettext_noop(u'Ibibio'), u'Ibibio'),
        'bo': (gettext_noop(u'Tibetan'), u'Bod skad'),
        'br': (gettext_noop(u'Breton'), u'Brezhoneg'),
        'bs': (gettext_noop(u'Bosnian'), u'Bosanski'),
        'ca': (gettext_noop(u'Catalan'), u'Català'),
        'ce': (gettext_noop(u'Chechen'), u'Chechen'),
        'ceb': (gettext_noop(u'Cebuano'), u'Cebuano'),
        'ch': (gettext_noop(u'Chamorro'), u'Chamoru'),
        'cho': (gettext_noop(u'Choctaw'), u'Choctaw'),
        'co': (gettext_noop(u'Corsican'), u'Corsu'),
        'cr': (gettext_noop(u'Cree'), u'Nehiyaw'),
        'cs': (gettext_noop(u'Czech'), u'Čeština'),
        'cu': (gettext_noop(u'Church Slavic'), u'Church Slavic'),
        'cv': (gettext_noop(u'Chuvash'), u'Chuvash'),
        'cy': (gettext_noop(u'Welsh'), u'Cymraeg'),
        'da': (gettext_noop(u'Danish'), u'Dansk'),
        'de': (gettext_noop(u'German'), u'Deutsch'),
        'dv': (gettext_noop(u'Divehi'), u'Divehi'),
        'dz': (gettext_noop(u'Dzongkha'), u'Dzongkha'),
        'ee': (gettext_noop(u'Ewe'), u'Ewe'),
        'efi': (gettext_noop(u'Efik'), u'Efik'),
        'el': (gettext_noop(u'Greek'), u'Ελληνικά'),
        'en': (gettext_noop(u'English'), u'English'),
        'en-gb': (gettext_noop(u'English, British'), u'English, British'),
        'eo': (gettext_noop(u'Esperanto'), u'Esperanto'),
        'es': (gettext_noop(u'Spanish'), u'Español'),
        'es-ar': (gettext_noop(u'Spanish, Argentinian'), u'Spanish, Argentinian'),
        'es-mx': (gettext_noop(u'Spanish, Mexican'), u'Spanish, Mexican'),
        'es-ni': (gettext_noop(u'Spanish, Nicaraguan'), u'Spanish, Nicaraguan, '),
        'et': (gettext_noop(u'Estonian'), u'Eesti'),
        'eu': (gettext_noop(u'Basque'), u'Euskara'),
        'fa': (gettext_noop(u'Persian'), u'فارسی'),
        'ff': (gettext_noop(u'Fulah'), u'Fulah'),
        'fi': (gettext_noop(u'Finnish'), u'Suomi'),
        'fil': (gettext_noop(u'Filipino'), u'Filipino'),
        'fj': (gettext_noop(u'Fijian'), u'Na Vosa Vakaviti'),
        'fo': (gettext_noop(u'Faroese'), u'Føroyskt'),
        'fr': (gettext_noop(u'French'), u'Français'),
        'fr-ca': (gettext_noop(u'French, Canadian'), u'French, Canadian'),
        'ful': (gettext_noop(u'Fula'), u'Fula'),
        'fy-nl': (gettext_noop(u'Frisian'), u'Frysk'),
{'fraction_non_alphanumeric': 0.1792320731078193, 'fraction_numerical': 0.005011423096764684, 'mean_word_length': 3.3563402889245584, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3276010', 'n_tokens_mistral': 5203, 'n_tokens_neox': 4928, 'n_words': 1199}