<commit_before># -*- coding: utf-8 -*-
#----------------------------------------------------------------------------#
# __init__.py
# <NAME> <<EMAIL>>
# vim: ts=4 sw=4 sts=4 et tw=78:
# Sun May 15 23:12:04 EST 2005
#
#----------------------------------------------------------------------------#

"""
This module is responsible for Japanese script/encoding specific methods,
especially determining the script type of an entry. It is thus the only
module which requires a utf8 encoding for the additional Japanese characters.
"""

#----------------------------------------------------------------------------#

import enum

#----------------------------------------------------------------------------#
# CONSTANTS
#----------------------------------------------------------------------------#

Script = enum.Enum(u'Hiragana', u'Katakana', u'Kanji', u'Ascii',
        u'FullAscii', u'Unknown')

_knownBands = {
        Script.Ascii:       (u'\u0021', u'\u00ff'),
        Script.Hiragana:	(u'\u3041', u'\u3096'),
        Script.Katakana:	(u'\u30a1', u'\u30f6'),
        Script.Kanji:	    (u'\u4e00', u'\u9fa5'),
        Script.FullAscii:	(u'\uff01', u'\uff5f'),
    }

_interKanaDistance = 96

#----------------------------------------------------------------------------#

class ScriptMapping:
    """
    A mapping function between two scripts. We assume that the given scripts
    are different versions of the same characters, and further that they are
    aligned at the start and end points stored.
    """
    def __init__(self, fromScript, toScript):
        """
        Constructor, initializes script conversion.
        
        @param fromScript: The script to convert from.
        @type fromScript: Script
        @param toScript: The script to convert to.
        @type toScript: Script
        """
        self.fromStart, self.fromEnd = _knownBands[fromScript]
        self.toStart, self.toEnd = _knownBands[toScript]
        self.ordDiff = ord(self.toStart) - ord(self.fromStart)
        assert (ord(self.fromEnd) - ord(self.fromStart)) <= \
                (ord(self.toEnd) - ord(self.toStart))
        return

    def __call__(self, jString):
        """
        Converts any matching characters in the given string between scripts.
        Any characters which don't match the input script are passed through
        unchanged.
        """
        result = []
        for char in jString:
            if self.fromStart <= char <= self.fromEnd:
                result.append(unichr(ord(char) + self.ordDiff))
            else:
                result.append(char)

        return u''.join(result)

toHiragana = ScriptMapping(Script.Katakana, Script.Hiragana)
toKatakana = ScriptMapping(Script.Hiragana, Script.Katakana)

#----------------------------------------------------------------------------#

_mappedChars = {
            u'\u3000':  u'\u0020',
            u'\u3001':  u'\u002c',
            u'\u3002':  u'\u002e',
        }

_toAscii = ScriptMapping(Script.FullAscii, Script.Ascii)

def normalizeAscii(jString):
    """
    Normalize a double-width ascii string, converting all characters to
    their single-width counterparts.

    @param jString: The string to convert.
    @type jString: unicode
    @return: The converted string.
    """
    result = []
    for char in jString:
        result.append(_mappedChars.get(char, char))

    return _toAscii(u''.join(result))

#----------------------------------------------------------------------------#

def getScript(script):
    """
    Returns a string containing all charcters in the given script.
    """

    scriptStarts, scriptEnds = _knownBands[script]

    output = []
    for i in xrange(ord(scriptStarts), ord(scriptEnds) + 1):
        output.append(unichr(i))

    return ''.join(output)

#----------------------------------------------------------------------------#

def compareKana(jStringA, jStringB):
    """
    Compares two kana strings in a script-independent manner.

        >>> ru_h = unicode('る', 'utf8')
        >>> ka_h = unicode('か', 'utf8')
        >>> ka_k = unicode('カ', 'utf8')
        >>> compareKana(ru_h, ka_k) == cmp(ru_h, ka_h)
        True
<commit_msg>Added a method to normalize half-width katakana, and full-width ascii jointly.<commit_after># -*- coding: utf-8 -*-
#----------------------------------------------------------------------------#
# __init__.py
# <NAME> <<EMAIL>>
# vim: ts=4 sw=4 sts=4 et tw=78:
# Sun May 15 23:12:04 EST 2005
#
#----------------------------------------------------------------------------#

"""
This module is responsible for Japanese script/encoding specific methods,
especially determining the script type of an entry. It is thus the only
module which requires a utf8 encoding for the additional Japanese characters.
"""

#----------------------------------------------------------------------------#

import enum

#----------------------------------------------------------------------------#
# CONSTANTS
#----------------------------------------------------------------------------#

Script = enum.Enum(u'Hiragana', u'Katakana', u'Kanji', u'Ascii',
        u'FullAscii', u'HalfKatakana', u'Unknown')

_knownBands = {
        Script.Ascii:           (u'\u0021', u'\u00ff'),
        Script.Hiragana:	    (u'\u3041', u'\u3096'),
        Script.Katakana:	    (u'\u30a1', u'\u30f6'),
        Script.Kanji:	        (u'\u4e00', u'\u9fa5'),
        Script.FullAscii:	    (u'\uff01', u'\uff5f'),
        Script.HalfKatakana:    (u'\uff61', u'\uff9f'),
    }

_interKanaDistance = 96

#----------------------------------------------------------------------------#

class ScriptMapping:
    """
    A mapping function between two scripts. We assume that the given scripts
    are different versions of the same characters, and further that they are
    aligned at the start and end points stored.
    """
    def __init__(self, fromScript, toScript):
        """
        Constructor, initializes script conversion.
        
        @param fromScript: The script to convert from.
        @type fromScript: Script
        @param toScript: The script to convert to.
        @type toScript: Script
        """
        self.fromStart, self.fromEnd = _knownBands[fromScript]
        self.toStart, self.toEnd = _knownBands[toScript]
        self.ordDiff = ord(self.toStart) - ord(self.fromStart)
        assert (ord(self.fromEnd) - ord(self.fromStart)) <= \
                (ord(self.toEnd) - ord(self.toStart))
        return

    def __call__(self, jString):
        """
        Converts any matching characters in the given string between scripts.
        Any characters which don't match the input script are passed through
        unchanged.
        """
        result = []
        for char in jString:
            if self.fromStart <= char <= self.fromEnd:
                result.append(unichr(ord(char) + self.ordDiff))
            else:
                result.append(char)

        return u''.join(result)

toHiragana = ScriptMapping(Script.Katakana, Script.Hiragana)
toKatakana = ScriptMapping(Script.Hiragana, Script.Katakana)

#----------------------------------------------------------------------------#

_mappedChars = {
            u'\u3000':  u'\u0020',
            u'\u3001':  u'\u002c',
            u'\u3002':  u'\u002e',
        }

_toAscii = ScriptMapping(Script.FullAscii, Script.Ascii)

def normalizeAscii(jString):
    """
    Normalize a double-width ascii string, converting all characters to
    their single-width counterparts.

    @param jString: The string to convert.
    @type jString: unicode
    @return: The converted string.
    """
    result = []
    for char in jString:
        result.append(_mappedChars.get(char, char))

    return _toAscii(u''.join(result))

#----------------------------------------------------------------------------#

_fullWidthKana = u"。「」、・ヲァィゥェォャュョッーアイウエオカキクケコサシスセソタチツテトナニヌネノハヒフヘホマミムメモヤユヨラリルレロワン゛゜"

def normalizeKana(jString):
    """
    Normalizes all half-width katakana to normal width katakana. Leaves
    all other characters unchanged.

        >>> x = normalizeKana(unicode('ｶｷｸｹｺ', 'utf8'))
        >>> x == unicode('カキクケコ', 'utf8') 
        True

    @param jString: The string to convert.
    @type jString: unicode
    @return: The converted string.
    """
    startHWKana, endHWKana = _knownBands[Script.HalfKatakana]
    result = []
    for char in jString:
        if startHWKana <= char <= endHWKana:
            result.append(_fullWidthKana[ord(char) - 0xff61])
        else:
            result.append(char)

    return u''.join(result)

#----------------------------------------------------------------------------#

def normalize(jString):
    """
    Jointly normalized full width ascii to single width ascii and half-width
    katakana to full with katakana.

        >>> x = normalize(unicode('Aあア阿ｱＡ', 'utf8'))
        >>> x == unicode('Aあア阿アA', 'utf8')
        True

    @param jString: The string to convert.
    @type jString: unicode
    @return: The converted string.
    """
    return normalizeAscii(normalizeKana(jString))

#----------------------------------------------------------------------------#

def getScript(script):
    """
    Returns a string containing all charcters in the given script.
    """

    scriptStarts, scriptEnds = _knownBands[script]

    output = []
    for i in xrange(ord(scriptStarts), ord(scriptEnds) + 1):
        output.append(unichr(i))

    return ''.join(output)

#----------------------------------------------------------------------------#

def compareKana(jStringA, jStringB):
    """
    Compares two kana strings in a script-independent manner.

        >>> ru_h = unicode('る', 'utf8')
        >>> ka_h = unicode('か', 'utf8')
        >>> ka_k = unicode('カ', 'utf8')
        >>> compareKana(ru_h, ka_k) == cmp(ru_h, ka_h)
        True

{'fraction_non_alphanumeric': 0.2597309965355614, 'fraction_numerical': 0.016303240269003465, 'mean_word_length': 1.5579880114672922, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5124827', 'n_tokens_mistral': 2947, 'n_tokens_neox': 2753, 'n_words': 705}