<commit_before># -*- coding: utf-8 -*-
#!/usr/bin/python

import unicodedata
import string


class TamilLetter:
    """ Class to handle Tamil letter processing

    The TamilLetter class has a number of different static functions that help
    with the processing of Tamil letters.
    """

    # A mapping between Tamil vowels and the grapheme that provides that
    # particular vowel sound if it were the 'extension'/'marking' of a
    # combination. E.g. 'ஆ' should map to 'ா'

    VOWEL_TO_ENDING_MAP = {
                            u'அ': u'',  # அ -> '' (empty string)
                            u'ஆ': u'\u0BBE',  # ஆ -> ா
                            u'இ': u'\u0BBF',  # இ -> ி
                            u'ஈ': u'\u0BC0',  # ஈ -> ீ
                            u'உ': u'\u0BC1',  # உ -> ு
                            u'ஊ': u'\u0BC2',  # ஊ -> ூ
                            u'எ': u'\u0BC6',  # எ -> ெ
                            u'ஏ': u'\u0BC7',  # ஏ -> ே
                            u'ஐ': u'\u0BC8',  # ஐ -> ை
                            u'ஒ': u'\u0BCA',  # ஒ -> ொ
                            u'ஓ': u'\u0BCB',  # ஓ -> ோ
                            u'ஔ': u'\u0BCC'  # ஔ -> ௌ
                            }

    # an inverse mapping of the above relationship, mapping  a grapheme that
    # represents the 'extension'/'marking' portion of a combination with its
    # respective vowel sound. E.g  'ா' should map to 'இ'
    ENDING_TO_VOWEL_MAP = dict((ending, vowel) for vowel, ending in
                            VOWEL_TO_ENDING_MAP.iteritems() if ending != u'')

    # create vowel ending list from the dictionary's keys and values
    COMBINATION_ENDINGS = ENDING_TO_VOWEL_MAP.keys()

    PULLI = u'்'

    AYTHAM = u'ஃ'

    CONSONANTS = {
                  'VALLINAM': (u'க்', u'ச்', u'ட்', u'த்', u'ப்', u'ற்'),
                  'MELLINAM': (u'ங்', u'ஞ்', u'ண்', u'ந்', u'ம்', u'ன்'),
                  'IDAIYINAM': (u'ய்', u'ர்', u'ல்', u'வ்', u'ழ்', u'ள்'),

                  # # Removing support for 'க்ஷ்' and 'ஶ்' temporarily
                  # TODO: Add support for 'க்ஷ்', 'ஶ்'
                  # 'GRANTHA'   : (u'ஶ்', u'ஜ்', u'ஷ்', u'ஸ்', u'ஹ்', u'க்ஷ்')
                  'GRANTHA': (u'ஜ்', u'ஷ்', u'ஸ்', u'ஹ்')
                  }

    VOWELS = {
              'KURIL': (u'அ', u'இ', u'உ', u'எ', u'ஒ'),
              'NEDIL': (u'ஆ', u'ஈ', u'ஊ', u'ஏ', u'ஐ', u'ஓ', u'ஔ')
              }

    @staticmethod
    def get_script_name(codepoint=u''):
        """ Returns name of character's script

        Returns the name of the script in which the unicode codepoint is
        encoded.

        Args:
            codepoint: A single unicode codepoint

        Returns:
            A string containing the name of the script that the unicode
            codepoint is encoded in. For example:
            for a Tamil codepoint, it would return 'Tamil', for a Western
            European codepoint it would return 'Latin', for a Russian codepoint
            it would return 'Cyrillic'

        Raises:
            TypeError: A single unicode codepoint was not entered
        """

        if not codepoint or not isinstance(codepoint, unicode) or \
            not (1 <= len(codepoint) <= 2):
                raise TypeError("Expecting single unicode character")

        # TODO: Sanity check - ensure all characters are same language

        return unicodedata.name(codepoint[0]).split()[0]

    @staticmethod
    def assert_valid_letter(letter=u''):
        """ Asserts that the given letter is a valid Tamil unicode grapheme

        Checks that the letter is of type unicode, is not equal to an empty
        string and is a Tamil letter (according to the Unicode consortium).

        Returns:
            True: If the letter is a valid Tamil unicode grapheme

        Raises:
            TypeError: Input was not of type unicode
            ValueError: Empty string, or non-Tamil grapheme or codepoint
        """

        # Check for non-unicode objects
        if not isinstance(letter, unicode):
            raise TypeError("""Must be unicode string. Value \'%s\' of type
                %s received""" % (letter, type(letter)))

        # Check for empty strings
        if len(letter) == 0:
            raise ValueError("Expected Unicode character, not empty string.")

        # Check for non-Tamil, non-whitespace characters
        script_language = TamilLetter.get_script_name(letter).title()
        if  script_language != 'Tamil' and \
            not TamilLetter.is_whitespace(letter):

            raise ValueError("""Expected a Tamil character or whitespace.
                \'%s\' is from the %s script""" % (letter, script_language))

        # TODO: Make this work if more than one character is entered

        # TODO: Allow punctuation and digits

        # If we've gotten this far, everything seems to be alright
        return True

    # # Getter methods for aytham, vowels, consonants and combinations
    @staticmethod
    def get_aytham():
        """ Returns Aytham (ஃ)

        Returns a single unicode codepoint - the Aytham - a Tamil letter
        that is neither a vowel, a consonant, or a combination.

        Returns:
            u'ஃ' - the Tamil letter 'aytham'
        """

        return TamilLetter.AYTHAM

    @staticmethod
    def get_vowels():
        """ Returns a tuple of all the vowels """

        # get keys from VOWEL_TO_ENDING_MAP and convert to tuple
        return tuple(TamilLetter.VOWEL_TO_ENDING_MAP.keys())

    @staticmethod
    def get_consonants():
        """ Returns a tuple of all the consonants """

        # merge all sub-iterables in CONSONANTS into one tuple
        return  tuple(consonant for consonant_group in TamilLetter.CONSONANTS.values() for consonant in consonant_group)

    @staticmethod
    def get_letter_type(letter=u''):
        """ Returns the letter type: Vowel, Consonant or Combination """

        # ensure that the letter is a valid single Tamil unicode grapheme
        TamilLetter.assert_valid_letter(letter)

        if TamilLetter.is_vowel(letter):
            return 'VOWEL'
        elif TamilLetter.is_consonant(letter):
            return 'CONSONANT'
        elif TamilLetter.is_combination(letter):
            return 'COMBINATION'
        elif TamilLetter.is_aytham(letter):
            return 'AYTHAM'
        else:<commit_msg>Made get_vowels() PEP8 and Google Style guide-compliant<commit_after># -*- coding: utf-8 -*-
#!/usr/bin/python

import unicodedata
import string


class TamilLetter:
    """ Class to handle Tamil letter processing

    The TamilLetter class has a number of different static functions that help
    with the processing of Tamil letters.
    """

    # A mapping between Tamil vowels and the grapheme that provides that
    # particular vowel sound if it were the 'extension'/'marking' of a
    # combination. E.g. 'ஆ' should map to 'ா'

    VOWEL_TO_ENDING_MAP = {
                            u'அ': u'',  # அ -> '' (empty string)
                            u'ஆ': u'\u0BBE',  # ஆ -> ா
                            u'இ': u'\u0BBF',  # இ -> ி
                            u'ஈ': u'\u0BC0',  # ஈ -> ீ
                            u'உ': u'\u0BC1',  # உ -> ு
                            u'ஊ': u'\u0BC2',  # ஊ -> ூ
                            u'எ': u'\u0BC6',  # எ -> ெ
                            u'ஏ': u'\u0BC7',  # ஏ -> ே
                            u'ஐ': u'\u0BC8',  # ஐ -> ை
                            u'ஒ': u'\u0BCA',  # ஒ -> ொ
                            u'ஓ': u'\u0BCB',  # ஓ -> ோ
                            u'ஔ': u'\u0BCC'  # ஔ -> ௌ
                            }

    # an inverse mapping of the above relationship, mapping  a grapheme that
    # represents the 'extension'/'marking' portion of a combination with its
    # respective vowel sound. E.g  'ா' should map to 'இ'
    ENDING_TO_VOWEL_MAP = dict((ending, vowel) for vowel, ending in
                            VOWEL_TO_ENDING_MAP.iteritems() if ending != u'')

    # create vowel ending list from the dictionary's keys and values
    COMBINATION_ENDINGS = ENDING_TO_VOWEL_MAP.keys()

    PULLI = u'்'

    AYTHAM = u'ஃ'

    CONSONANTS = {
                  'VALLINAM': (u'க்', u'ச்', u'ட்', u'த்', u'ப்', u'ற்'),
                  'MELLINAM': (u'ங்', u'ஞ்', u'ண்', u'ந்', u'ம்', u'ன்'),
                  'IDAIYINAM': (u'ய்', u'ர்', u'ல்', u'வ்', u'ழ்', u'ள்'),

                  # # Removing support for 'க்ஷ்' and 'ஶ்' temporarily
                  # TODO: Add support for 'க்ஷ்', 'ஶ்'
                  # 'GRANTHA'   : (u'ஶ்', u'ஜ்', u'ஷ்', u'ஸ்', u'ஹ்', u'க்ஷ்')
                  'GRANTHA': (u'ஜ்', u'ஷ்', u'ஸ்', u'ஹ்')
                  }

    VOWELS = {
              'KURIL': (u'அ', u'இ', u'உ', u'எ', u'ஒ'),
              'NEDIL': (u'ஆ', u'ஈ', u'ஊ', u'ஏ', u'ஐ', u'ஓ', u'ஔ')
              }

    @staticmethod
    def get_script_name(codepoint=u''):
        """ Returns name of character's script

        Returns the name of the script in which the unicode codepoint is
        encoded.

        Args:
            codepoint: A single unicode codepoint

        Returns:
            A string containing the name of the script that the unicode
            codepoint is encoded in. For example:
            for a Tamil codepoint, it would return 'Tamil', for a Western
            European codepoint it would return 'Latin', for a Russian codepoint
            it would return 'Cyrillic'

        Raises:
            TypeError: A single unicode codepoint was not entered
        """

        if not codepoint or not isinstance(codepoint, unicode) or \
            not (1 <= len(codepoint) <= 2):
                raise TypeError("Expecting single unicode character")

        # TODO: Sanity check - ensure all characters are same language

        return unicodedata.name(codepoint[0]).split()[0]

    @staticmethod
    def assert_valid_letter(letter=u''):
        """ Asserts that the given letter is a valid Tamil unicode grapheme

        Checks that the letter is of type unicode, is not equal to an empty
        string and is a Tamil letter (according to the Unicode consortium).

        Returns:
            True: If the letter is a valid Tamil unicode grapheme

        Raises:
            TypeError: Input was not of type unicode
            ValueError: Empty string, or non-Tamil grapheme or codepoint
        """

        # Check for non-unicode objects
        if not isinstance(letter, unicode):
            raise TypeError("""Must be unicode string. Value \'%s\' of type
                %s received""" % (letter, type(letter)))

        # Check for empty strings
        if len(letter) == 0:
            raise ValueError("Expected Unicode character, not empty string.")

        # Check for non-Tamil, non-whitespace characters
        script_language = TamilLetter.get_script_name(letter).title()
        if  script_language != 'Tamil' and \
            not TamilLetter.is_whitespace(letter):

            raise ValueError("""Expected a Tamil character or whitespace.
                \'%s\' is from the %s script""" % (letter, script_language))

        # TODO: Make this work if more than one character is entered

        # TODO: Allow punctuation and digits

        # If we've gotten this far, everything seems to be alright
        return True

    # # Getter methods for aytham, vowels, consonants and combinations
    @staticmethod
    def get_aytham():
        """ Returns Aytham (ஃ)

        Returns a single unicode codepoint - the Aytham - a Tamil letter
        that is neither a vowel, a consonant, or a combination.

        Returns:
            u'ஃ' - the Tamil letter 'aytham'
        """

        return TamilLetter.AYTHAM

    @staticmethod
    def get_vowels():
        """ Returns a tuple of all the Tamil vowels

        Returns a single tuple containing all twelve Tamil vowels:
        (அ, ஆ, இ, ஈ, உ, ஊ, எ, ஏ, ஐ, ஒ, ஓ, ஔ)

        Returns:
            A tuple containing all 12 Tamil vowels
        """

        # get keys from VOWEL_TO_ENDING_MAP and convert to tuple
        return tuple(TamilLetter.VOWEL_TO_ENDING_MAP.keys())

    @staticmethod
    def get_consonants():
        """ Returns a tuple of all the consonants """

        # merge all sub-iterables in CONSONANTS into one tuple
        return  tuple(consonant for consonant_group in TamilLetter.CONSONANTS.values() for consonant in consonant_group)

    @staticmethod
    def get_letter_type(letter=u''):
        """ Returns the letter type: Vowel, Consonant or Combination """

        # ensure that the letter is a valid single Tamil unicode grapheme
        TamilLetter.assert_valid_letter(letter)

        if TamilLetter.is_vowel(letter):
            return 'VOWEL'
        elif TamilLetter.is_consonant(letter):
            return 'CONSONANT'
        elif TamilLetter.is_combination(letter):
            return 'COMBINATION'
        elif TamilLetter.is_aytham(letter):
            return 'AYTHAM'
        else:
{'fraction_non_alphanumeric': 0.11023500309214596, 'fraction_numerical': 0.003787878787878788, 'mean_word_length': 1.8596374889478338, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1421255', 'n_tokens_mistral': 4390, 'n_tokens_neox': 4005, 'n_words': 1283}