<commit_before>import re
import six

import phonenumbers

from postal.near_dupe import near_dupe_hashes
from postal.dedupe import place_languages, duplicate_status, is_name_duplicate, is_street_duplicate, is_house_number_duplicate, is_po_box_duplicate, is_unit_duplicate, is_floor_duplicate, is_postal_code_duplicate, is_toponym_duplicate, is_name_duplicate_fuzzy, is_street_duplicate_fuzzy
from postal.normalize import normalized_tokens, DEFAULT_STRING_OPTIONS, NORMALIZE_STRING_REPLACE_NUMEX
from postal.tokenize import tokenize
from postal.token_types import token_types

from lieu.address import AddressComponents, EntityDetails, Coordinates
from lieu.api import DedupeResponse
from lieu.word_index import WordIndex

whitespace_regex = re.compile('[\s]+')


class AddressDeduper(object):
    DEFAULT_GEOHASH_PRECISION = 7

    address_only_keys = True
    name_only_keys = False
    name_and_address_keys = False
    with_name = False

    us_zip5_pattern = re.compile('^\s*([0-9]{5})\-?([0-9]{4})\s*$')

    @classmethod
    def address_dupe_status(cls, a1, a2, languages=None, fuzzy_street_name=False):
        a1_street = a1.get(AddressComponents.STREET)
        a2_street = a2.get(AddressComponents.STREET)

        if a1_street:
            a1_street = a1_street.strip()
        if a2_street:
            a2_street = a2_street.strip()

        a1_house_number = a1.get(AddressComponents.HOUSE_NUMBER)
        a2_house_number = a2.get(AddressComponents.HOUSE_NUMBER)

        if a1_house_number:
            a1_house_number = a1_house_number.strip()
        if a2_house_number:
            a2_house_number = a2_house_number.strip()

        a1_base_house_number = a1.get(AddressComponents.HOUSE_NUMBER_BASE)
        a2_base_house_number = a2.get(AddressComponents.HOUSE_NUMBER_BASE)

        if a1_base_house_number:
            a1_base_house_number = a1_base_house_number.strip()
        if a2_base_house_number:
            a2_base_house_number = a2_base_house_number.strip()

        if (a1_street and not a2_street) or (a2_street and not a1_street):
            return (duplicate_status.NON_DUPLICATE, 0.0)

        if (a1_house_number and not a2_house_number) or (a2_house_number and not a1_house_number):
            return (duplicate_status.NON_DUPLICATE, 0.0)

        have_street = a1_street and a2_street
        same_street = False
        street_status = duplicate_status.NON_DUPLICATE

        street_sim = 0.0
        if have_street:
            street_status = is_street_duplicate(a1_street, a2_street, languages=languages)
            same_street = street_status in (duplicate_status.EXACT_DUPLICATE, duplicate_status.LIKELY_DUPLICATE)
            if street_status == duplicate_status.EXACT_DUPLICATE:
                street_sim = 1.0
            elif street_status == duplicate_status.NEEDS_REVIEW:
                street_sim = 0.5
            elif street_status == duplicate_status.LIKELY_DUPLICATE:
                street_sim = 0.9
            else:
                street_sim = 0.0
            if not same_street and fuzzy_street_name:
                a1_street_tokens = Name.content_tokens(a1_street, languages=languages)
                a1_scores_norm = WordIndex.normalized_vector([1] * len(a1_street_tokens))
                a2_street_tokens = Name.content_tokens(a2_street, languages=languages)
                a2_scores_norm = WordIndex.normalized_vector([1] * len(a2_street_tokens))
                street_status, street_sim = is_street_duplicate_fuzzy(a1_street_tokens, a1_scores_norm, a2_street_tokens, a2_scores_norm, languages=languages)
                same_street = street_status in (duplicate_status.EXACT_DUPLICATE, duplicate_status.LIKELY_DUPLICATE)
            if not same_street:
                return (duplicate_status.NON_DUPLICATE, 0.0)

        have_house_number = a1_house_number and a2_house_number
        have_base_house_number = a1_base_house_number or a2_base_house_number
        same_house_number = False
        house_number_status = duplicate_status.NON_DUPLICATE

        if have_house_number:
            house_number_sim = 0.0
            house_number_status = is_house_number_duplicate(a1_house_number, a2_house_number, languages=languages)
            same_house_number = house_number_status == duplicate_status.EXACT_DUPLICATE
            if same_house_number:
                house_number_sim = 1.0<commit_msg>[fix] declare house_number_sim outside of conditional<commit_after>import re
import six

import phonenumbers

from postal.near_dupe import near_dupe_hashes
from postal.dedupe import place_languages, duplicate_status, is_name_duplicate, is_street_duplicate, is_house_number_duplicate, is_po_box_duplicate, is_unit_duplicate, is_floor_duplicate, is_postal_code_duplicate, is_toponym_duplicate, is_name_duplicate_fuzzy, is_street_duplicate_fuzzy
from postal.normalize import normalized_tokens, DEFAULT_STRING_OPTIONS, NORMALIZE_STRING_REPLACE_NUMEX
from postal.tokenize import tokenize
from postal.token_types import token_types

from lieu.address import AddressComponents, EntityDetails, Coordinates
from lieu.api import DedupeResponse
from lieu.word_index import WordIndex

whitespace_regex = re.compile('[\s]+')


class AddressDeduper(object):
    DEFAULT_GEOHASH_PRECISION = 7

    address_only_keys = True
    name_only_keys = False
    name_and_address_keys = False
    with_name = False

    us_zip5_pattern = re.compile('^\s*([0-9]{5})\-?([0-9]{4})\s*$')

    @classmethod
    def address_dupe_status(cls, a1, a2, languages=None, fuzzy_street_name=False):
        a1_street = a1.get(AddressComponents.STREET)
        a2_street = a2.get(AddressComponents.STREET)

        if a1_street:
            a1_street = a1_street.strip()
        if a2_street:
            a2_street = a2_street.strip()

        a1_house_number = a1.get(AddressComponents.HOUSE_NUMBER)
        a2_house_number = a2.get(AddressComponents.HOUSE_NUMBER)

        if a1_house_number:
            a1_house_number = a1_house_number.strip()
        if a2_house_number:
            a2_house_number = a2_house_number.strip()

        a1_base_house_number = a1.get(AddressComponents.HOUSE_NUMBER_BASE)
        a2_base_house_number = a2.get(AddressComponents.HOUSE_NUMBER_BASE)

        if a1_base_house_number:
            a1_base_house_number = a1_base_house_number.strip()
        if a2_base_house_number:
            a2_base_house_number = a2_base_house_number.strip()

        if (a1_street and not a2_street) or (a2_street and not a1_street):
            return (duplicate_status.NON_DUPLICATE, 0.0)

        if (a1_house_number and not a2_house_number) or (a2_house_number and not a1_house_number):
            return (duplicate_status.NON_DUPLICATE, 0.0)

        have_street = a1_street and a2_street
        same_street = False
        street_status = duplicate_status.NON_DUPLICATE

        street_sim = 0.0
        if have_street:
            street_status = is_street_duplicate(a1_street, a2_street, languages=languages)
            same_street = street_status in (duplicate_status.EXACT_DUPLICATE, duplicate_status.LIKELY_DUPLICATE)
            if street_status == duplicate_status.EXACT_DUPLICATE:
                street_sim = 1.0
            elif street_status == duplicate_status.NEEDS_REVIEW:
                street_sim = 0.5
            elif street_status == duplicate_status.LIKELY_DUPLICATE:
                street_sim = 0.9
            else:
                street_sim = 0.0
            if not same_street and fuzzy_street_name:
                a1_street_tokens = Name.content_tokens(a1_street, languages=languages)
                a1_scores_norm = WordIndex.normalized_vector([1] * len(a1_street_tokens))
                a2_street_tokens = Name.content_tokens(a2_street, languages=languages)
                a2_scores_norm = WordIndex.normalized_vector([1] * len(a2_street_tokens))
                street_status, street_sim = is_street_duplicate_fuzzy(a1_street_tokens, a1_scores_norm, a2_street_tokens, a2_scores_norm, languages=languages)
                same_street = street_status in (duplicate_status.EXACT_DUPLICATE, duplicate_status.LIKELY_DUPLICATE)
            if not same_street:
                return (duplicate_status.NON_DUPLICATE, 0.0)

        have_house_number = a1_house_number and a2_house_number
        have_base_house_number = a1_base_house_number or a2_base_house_number
        same_house_number = False
        house_number_status = duplicate_status.NON_DUPLICATE
        house_number_sim = 0.0

        if have_house_number:
            house_number_status = is_house_number_duplicate(a1_house_number, a2_house_number, languages=languages)
            same_house_number = house_number_status == duplicate_status.EXACT_DUPLICATE
            if same_house_number:
                house_number_sim = 1.0
{'fraction_non_alphanumeric': 0.06301182893539581, 'fraction_numerical': 0.020928116469517744, 'mean_word_length': 3.2850877192982457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '763264', 'n_tokens_mistral': 3023, 'n_tokens_neox': 2820, 'n_words': 482}