<commit_before>#!/usr/bin/python
# -*- coding: UTF-8 -*-
from HTMLParser import HTMLParser


def tuple_to_dict(lst):
    out = {}
    for tpl in lst:
        out[tpl[0]] = tpl[1]

    return out

class PrintableList(list):
    def __str__(self):
        out = '['
        out += ', '.join(self)
        out += ']'
        return out


class Reading(object):

    TYPE_END = 'ending'
    TYPE_OPT = 'optional'

    TYPES = (
        TYPE_END,
        TYPE_OPT,
    )

    COLOUR_END = u'#4b60ff'
    COLOUR_OPT = u'#e4240d'

    MAPPING = {
        COLOUR_END: TYPE_END,
        COLOUR_OPT: TYPE_OPT,
    }

    class Error(Exception):
        pass

    # create a subclass and override the handler methods
    class KanjiParser(HTMLParser):

        @staticmethod
        def attr_to_type(attrs):
            type = attrs.get('color', False)
            if not type:
                return False

            type = Reading.MAPPING.get(type, False)
            if not type:
                return False

            return type


        def handle_data(self, data):
            self.reading.append(data, type=self._type)

        def handle_endtag(self, tag):
            if tag != 'font':
                raise Reading.Error("Invalid Reading End Tag <%s>" % tag)

            self._reset_type()

        def handle_starttag(self, tag, attrs):
            attrs = tuple_to_dict(attrs)

            # Validate the tag
            if tag != 'font':
                raise Reading.Error("Invalid Reading Tag <%s>" % tag)

            # Validate the type
            type = self.attr_to_type(attrs)
            if not type:
                return

            # Validate the state change
            if self._type is not None:
                raise Reading.Error("Invalid State Change, nested type")

            # Now we can do the actual state change
            self._set_type(type)<commit_msg>Added katakana to readings<commit_after>#!/usr/bin/python
# -*- coding: UTF-8 -*-
from HTMLParser import HTMLParser


def tuple_to_dict(lst):
    out = {}
    for tpl in lst:
        out[tpl[0]] = tpl[1]

    return out

class PrintableList(list):
    def __str__(self):
        out = '['
        out += ', '.join(self)
        out += ']'
        return out

__by_vowels = {
    u'ア': u'ワラヤャマハナタサカアァ',
    u'イ': u'リミヒニちシキイィ',
    u'ウ': u'ルユュムフヌツスクウゥ',
    u'エ': u'レメヘネテセケエェ',
    u'オ': u'ヲロヨョモホノトソコオォ',
}

__to_vowel = {}
for vowel, letters in __by_vowels.iteritems():
    for letter in letters:
        __to_vowel[letter] = vowel

from jcconv import kata2hira, hira2kata

def char_to_base_vowel(char):
    translated = __to_vowel.get(char, False) or __to_vowel.get(hira2kata(char), False)

    if translated is False:
        raise Exception("Can't convert")

    return translated


def all_to_hiragana(string):
    out = u''

    for index, char in enumerate(string):
        if char == u'ー':
            char = char_to_base_vowel(out[-1])

        char = kata2hira(char)

        out += char

    return out



class Reading(object):

    TYPE_END = 'ending'
    TYPE_OPT = 'optional'

    TYPES = (
        TYPE_END,
        TYPE_OPT,
    )

    COLOUR_END = u'#4b60ff'
    COLOUR_OPT = u'#e4240d'

    MAPPING = {
        COLOUR_END: TYPE_END,
        COLOUR_OPT: TYPE_OPT,
    }

    class Error(Exception):
        pass

    # create a subclass and override the handler methods
    class KanjiParser(HTMLParser):

        @staticmethod
        def attr_to_type(attrs):
            type = attrs.get('color', False)
            if not type:
                return False

            type = Reading.MAPPING.get(type, False)
            if not type:
                return False

            return type


        def handle_data(self, data):
            self.reading.append(all_to_hiragana(data), type=self._type)

        def handle_endtag(self, tag):
            if tag != 'font':
                raise Reading.Error("Invalid Reading End Tag <%s>" % tag)

            self._reset_type()

        def handle_starttag(self, tag, attrs):
            attrs = tuple_to_dict(attrs)

            # Validate the tag
            if tag != 'font':
                raise Reading.Error("Invalid Reading Tag <%s>" % tag)

            # Validate the type
            type = self.attr_to_type(attrs)
            if not type:
                return

            # Validate the state change
            if self._type is not None:
                raise Reading.Error("Invalid State Change, nested type")

            # Now we can do the actual state change
            self._set_type(type)
{'fraction_non_alphanumeric': 0.10063443447823234, 'fraction_numerical': 0.005469262743382192, 'mean_word_length': 1.784409257003654, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5140945', 'n_tokens_mistral': 1509, 'n_tokens_neox': 1437, 'n_words': 367}