<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re

class Address(object):

    TOKEN_RE = re.compile(u'''
        (?:
            (?P<no>\d+)
            (?:[之-](?P<subno>\d+))?
            |
            (?P<name>.+?)
        )
        (?P<unit>[縣市鄉鎮市區村里路街巷弄號樓])
    ''', re.X)

    NO = 0
    SUBNO = 1
    NAME = 2
    UNIT = 3

    @staticmethod
    def tokenize(addr_str):
        if isinstance(addr_str, str):
            addr_str = addr_str.decode('utf-8')
        addr_str = addr_str.replace(u' ', '').replace(u'　', '')
        return tuple(Address.TOKEN_RE.findall(addr_str))

    @staticmethod
    def extract_no_pair(token):
        return (
            int(token[Address.NO]     or 0),
            int(token[Address.SUBNO] or 0)
        )

    def __init__(self, addr_str):

        self.addr_str = addr_str
        self.tokens = Address.tokenize(addr_str)
        self.first_no_token_idx = 0
        self.number_pair = (0, 0)

        len_tokens = len(self.tokens)
        while self.first_no_token_idx < len_tokens:
            if self.tokens[self.first_no_token_idx][Address.NO]:
                self.number_pair = Address.extract_no_pair(self.tokens[self.first_no_token_idx])
                break
            self.first_no_token_idx += 1

    def __repr__(self):
        return 'Address(%r)' % self.addr_str

    def __cmp__(self, other):
<commit_msg>handle commas<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import re

class Address(object):

    TOKEN_RE = re.compile(u'''
        (?:
            (?P<no>\d+)
            (?:[之-](?P<subno>\d+))?
            |
            (?P<name>.+?)
        )
        (?P<unit>[縣市鄉鎮市區村里路街巷弄號樓])
    ''', re.X)

    NO = 0
    SUBNO = 1
    NAME = 2
    UNIT = 3

    @staticmethod
    def tokenize(addr_str):
        if isinstance(addr_str, str):
            addr_str = addr_str.decode('utf-8')
        addr_str = addr_str.replace(u' ', u'').replace(u'　', u'').replace(u',', u'').replace(u'，', u'')
        return tuple(Address.TOKEN_RE.findall(addr_str))

    @staticmethod
    def extract_no_pair(token):
        return (
            int(token[Address.NO]     or 0),
            int(token[Address.SUBNO] or 0)
        )

    def __init__(self, addr_str):

        self.addr_str = addr_str
        self.tokens = Address.tokenize(addr_str)
        self.first_no_token_idx = 0
        self.number_pair = (0, 0)

        len_tokens = len(self.tokens)
        while self.first_no_token_idx < len_tokens:
            if self.tokens[self.first_no_token_idx][Address.NO]:
                self.number_pair = Address.extract_no_pair(self.tokens[self.first_no_token_idx])
                break
            self.first_no_token_idx += 1

    def __repr__(self):
        return 'Address(%r)' % self.addr_str

    def __cmp__(self, other):

{'fraction_non_alphanumeric': 0.1426543647977289, 'fraction_numerical': 0.008516678495386799, 'mean_word_length': 1.9736286919831223, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7175482', 'n_tokens_mistral': 1055, 'n_tokens_neox': 994, 'n_words': 160}