<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__version__ = '0.3'

import re

class Address(object):

    TOKEN_RE = re.compile(u'''
        (?:
            (?P<no>\d+)
            (?P<subno>之\d+)?
            (?=[巷弄號樓])
            |
            (?P<name>.+?)
        )
        (?:
            (?P<unit>[縣市鄉鎮市區村里路段街巷弄號樓])
            |
            (?=\d+(?:[之-]\d+)?[巷弄號樓]|$)
        )
    ''', re.X)

    NO    = 0
    SUBNO = 1
    NAME  = 2
    UNIT  = 3

    TO_REPLACE_RE = re.compile(u'''
    [ 　,，台-]
    |
    [０-９]
    |
    [一二三四五六七八九]?
    十?
    [一二三四五六七八九]
    (?=[段路街巷弄號樓])
    ''', re.X)

    TO_REMOVE_SET = set(u' 　,，')

    TO_REPLACE_MAP = {
        u'-' : u'之', u'台': u'臺',
        u'一': u'1', u'二': u'2', u'三': u'3', u'四': u'4', u'五': u'5',
        u'六': u'6', u'七': u'7', u'八': u'8', u'九': u'9',
    }

    @staticmethod
    def normalize(s):

        if isinstance(s, str):
            s = s.decode('utf-8')

        def replace(m):

            found = m.group()

            if found in Address.TO_REMOVE_SET:
                return u''
            if found in Address.TO_REPLACE_MAP:
                return Address.TO_REPLACE_MAP[found]

            len_found = len(found)

            # 65296 = '０'; 65305 = '９'; 65248 = '０'-'0'
            if len_found == 1 and 65296 <= ord(found) <= 65305:
                return unichr(ord(found)-65248)

            # for '十一' to '九十九'
            if len_found == 2:
                return u'1'+Address.TO_REPLACE_MAP[found[1]]
            if len_found == 3:
                return Address.TO_REPLACE_MAP[found[0]]+Address.TO_REPLACE_MAP[found[2]]

        s = Address.TO_REPLACE_RE.sub(replace, s)

        return s

    @staticmethod
    def tokenize(addr_str):
        return tuple(Address.TOKEN_RE.findall(Address.normalize(addr_str)))

    def __init__(self, addr_str):
        self.tokens = Address.tokenize(addr_str)

    @staticmethod
    def flat_tokens(tokens, n):<commit_msg>made flat_tokens's n optional<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

__version__ = '0.3'

import re

class Address(object):

    TOKEN_RE = re.compile(u'''
        (?:
            (?P<no>\d+)
            (?P<subno>之\d+)?
            (?=[巷弄號樓])
            |
            (?P<name>.+?)
        )
        (?:
            (?P<unit>[縣市鄉鎮市區村里路段街巷弄號樓])
            |
            (?=\d+(?:[之-]\d+)?[巷弄號樓]|$)
        )
    ''', re.X)

    NO    = 0
    SUBNO = 1
    NAME  = 2
    UNIT  = 3

    TO_REPLACE_RE = re.compile(u'''
    [ 　,，台-]
    |
    [０-９]
    |
    [一二三四五六七八九]?
    十?
    [一二三四五六七八九]
    (?=[段路街巷弄號樓])
    ''', re.X)

    TO_REMOVE_SET = set(u' 　,，')

    TO_REPLACE_MAP = {
        u'-' : u'之', u'台': u'臺',
        u'一': u'1', u'二': u'2', u'三': u'3', u'四': u'4', u'五': u'5',
        u'六': u'6', u'七': u'7', u'八': u'8', u'九': u'9',
    }

    @staticmethod
    def normalize(s):

        if isinstance(s, str):
            s = s.decode('utf-8')

        def replace(m):

            found = m.group()

            if found in Address.TO_REMOVE_SET:
                return u''
            if found in Address.TO_REPLACE_MAP:
                return Address.TO_REPLACE_MAP[found]

            len_found = len(found)

            # 65296 = '０'; 65305 = '９'; 65248 = '０'-'0'
            if len_found == 1 and 65296 <= ord(found) <= 65305:
                return unichr(ord(found)-65248)

            # for '十一' to '九十九'
            if len_found == 2:
                return u'1'+Address.TO_REPLACE_MAP[found[1]]
            if len_found == 3:
                return Address.TO_REPLACE_MAP[found[0]]+Address.TO_REPLACE_MAP[found[2]]

        s = Address.TO_REPLACE_RE.sub(replace, s)

        return s

    @staticmethod
    def tokenize(addr_str):
        return tuple(Address.TOKEN_RE.findall(Address.normalize(addr_str)))

    def __init__(self, addr_str):
        self.tokens = Address.tokenize(addr_str)

    @staticmethod
    def flat_tokens(tokens, n=None):
{'fraction_non_alphanumeric': 0.17386909274703058, 'fraction_numerical': 0.030326004548900682, 'mean_word_length': 1.6386666666666667, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4173390', 'n_tokens_mistral': 1768, 'n_tokens_neox': 1654, 'n_words': 272}