<commit_before># Meant to parse out address lines, minus city,state,zip into a usable dict for address matching
# Ignores periods and commas, because no one cares.

import re
import csv
import os
import dstk
import sys

# Keep lowercase, no periods
# Requires numbers first, then option dash plus numbers.
street_num_regex = r'^(\d+)([-/]?)(\d*)$'

apartment_regex_number = r'(#?)(\d*)(\w*)'
cwd = os.path.dirname(os.path.realpath(__file__))


class AddressParser(object):
    """
    AddressParser will be use to create Address objects. It contains a list of preseeded cities, states, prefixes,
    suffixes, and street names that will help the Address object correctly parse the given string. It is loaded
    with defaults that work in the average case, but can be adjusted for specific cases.
    """
    suffixes = {}
    # Lower case list of cities, used as a hint
    cities = []
    # Lower case list of streets, used as a hint
    streets = []
    prefixes = {
        "n": "N.", "e": "E.", "s": "S.", "w": "W.", "ne": "NE.", "nw": "NW.", 'se': "SE.", 'sw': "SW.", 'north': "N.",
        'east': "E.", 'south': "S.",
        'west': "W.", 'northeast': "NE.", 'northwest': "NW.", 'southeast': "SE.", 'southwest': "SW."}
    states = {
        'Mississippi': 'MS', 'Oklahoma': 'OK', 'Delaware': 'DE', 'Minnesota': 'MN', 'Illinois': 'IL', 'Arkansas': 'AR',
        'New Mexico': 'NM', 'Indiana': 'IN', 'Maryland': 'MD', 'Louisiana': 'LA', 'Idaho': 'ID', 'Wyoming': 'WY',
        'Tennessee': 'TN', 'Arizona': 'AZ', 'Iowa': 'IA', 'Michigan': 'MI', 'Kansas': 'KS', 'Utah': 'UT',
        'Virginia': 'VA', 'Oregon': 'OR', 'Connecticut': 'CT', 'Montana': 'MT', 'California': 'CA',
        'Massachusetts': 'MA', 'West Virginia': 'WV', 'South Carolina': 'SC', 'New Hampshire': 'NH',
        'Wisconsin': 'WI', 'Vermont': 'VT', 'Georgia': 'GA', 'North Dakota': 'ND', 'Pennsylvania': 'PA',
        'Florida': 'FL', 'Alaska': 'AK', 'Kentucky': 'KY', 'Hawaii': 'HI', 'Nebraska': 'NE', 'Missouri': 'MO',
        'Ohio': 'OH', 'Alabama': 'AL', 'New York': 'NY', 'South Dakota': 'SD', 'Colorado': 'CO', 'New Jersey': 'NJ',
        'Washington': 'WA', 'North Carolina': 'NC', 'District of Columbia': 'DC', 'Texas': 'TX', 'Nevada': 'NV',
        'Maine': 'ME', 'Rhode Island': 'RI'}

    def __init__(self, suffixes=None, cities=None, streets=None, backend="default", dstk_api_base=None, logger=None, required_confidence=0.65):
        """
        suffixes, cities and streets provide a chance to use different lists than the provided lists.
        suffixes is probably good for most users, unless you have some suffixes not recognized by USPS.
        cities is a very expansive list that may lead to false positives in some cases. If you only have a few cities
        you know will show up, provide your own list for better accuracy. If you are doing addresses across the US,
        the provided list is probably better.
        streets can be used to limit the list of possible streets the address are on. It comes blank by default and
        uses positional clues instead. If you are instead just doing a couple cities, a list of all possible streets
        will decrease incorrect street names.
        Valid backends include "default" and "dstk". If backend is dstk, it requires a dstk_api_base. Example of
        dstk_api_base would be 'http://example.com'.
        """
        self.logger = logger
        self.backend = backend
        self.dstk_api_base = dstk_api_base
        self.required_confidence = required_confidence
        if suffixes:
            self.suffixes = suffixes
        else:
            self.load_suffixes(os.path.join(cwd, "suffixes.csv"))
        if cities:
            self.cities = cities
        else:
            self.load_cities(os.path.join(cwd, "cities.csv"))
        if streets:
            self.streets = streets
        else:
            self.load_streets(os.path.join(cwd, "streets.csv"))
        if backend == "dstk":
            if dstk_api_base is None:
                raise ValueError("dstk_api_base is required for dstk backend.")
            self.dstk = dstk.DSTK({'apiBase': dstk_api_base})
        elif backend == "default":
            pass
        else:
            raise ValueError("backend must be either 'default' or 'dstk'.")

    def parse_address(self, address, line_number=-1):
        """
        Return an Address object from the given address. Passes itself to the Address constructor to use all the custom
        loaded suffixes, cities, etc.
        """
        return Address(address, self, line_number, self.logger)

    def dstk_multi_address(self, address_list):
        if self.backend != "dstk":
            raise ValueError("Only allowed for DSTK backends.")
        if self.logger: self.logger.debug("Sending {0} possible addresses to DSTK".format(len(address_list)))
        multi_address = self.dstk.street2coordinates(address_list)
        if self.logger: self.logger.debug("Received {0} addresses from DSTK".format(len(multi_address)))
        # if self.logger: self.logger.debug("End street2coords")
        addresses = []
        # if self.logger: self.logger.debug("Multi Addresses: {0}".format(multi_address))
        for address, dstk_return in multi_address.items():
            try:
                if dstk_return is None:
                    # if self.logger: self.logger.debug("DSTK None return for: {0}".format(address))
                    continue
                addresses.append(Address(address, self, -1, self.logger, dstk_pre_parse=dstk_return))
                if self.logger: self.logger.debug("DSTK Address Appended: {0}".format(dstk_return))
            except InvalidAddressException as e:
                # if self.logger: self.logger.debug("Error from dstk Address: {0}".format(e.message))
                continue
            except DSTKConfidenceTooLowException as e:
                continue
        return addresses


    def load_suffixes(self, filename):
        """
        Build the suffix dictionary. The keys will be possible long versions, and the values will be the
        accepted abbreviations. Everything should be stored using the value version, and you can search all
        by using building a set of self.suffixes.keys() and self.suffixes.values().
        """
        with open(filename, 'r') as f:
            for line in f:
                # Make sure we have key and value
                if len(line.split(',')) != 2:
                    continue
                    # Strip off newlines.
                self.suffixes[line.strip().split(',')[0]] = line.strip().split(',')[1]

    def load_cities(self, filename):
        """
        Load up all cities in lowercase for easier matching. The file should have one city per line, with no extra
        characters. This isn't strictly required, but will vastly increase the accuracy.<commit_msg>Solving issue #1 in forked repo by @GrammyPuter: Please Export AddressException<commit_after># Meant to parse out address lines, minus city,state,zip into a usable dict for address matching
# Ignores periods and commas, because no one cares.

import re
import csv
import os
import dstk
import sys

# Keep lowercase, no periods
# Requires numbers first, then option dash plus numbers.
street_num_regex = r'^(\d+)([-/]?)(\d*)$'

apartment_regex_number = r'(#?)(\d*)(\w*)'
cwd = os.path.dirname(os.path.realpath(__file__))


class AddressParser(object):
    """
    AddressParser will be use to create Address objects. It contains a list of preseeded cities, states, prefixes,
    suffixes, and street names that will help the Address object correctly parse the given string. It is loaded
    with defaults that work in the average case, but can be adjusted for specific cases.
    """
    suffixes = {}
    # Lower case list of cities, used as a hint
    cities = []
    # Lower case list of streets, used as a hint
    streets = []
    prefixes = {
        "n": "N.", "e": "E.", "s": "S.", "w": "W.", "ne": "NE.", "nw": "NW.", 'se': "SE.", 'sw': "SW.", 'north': "N.",
        'east': "E.", 'south': "S.",
        'west': "W.", 'northeast': "NE.", 'northwest': "NW.", 'southeast': "SE.", 'southwest': "SW."}
    states = {
        'Mississippi': 'MS', 'Oklahoma': 'OK', 'Delaware': 'DE', 'Minnesota': 'MN', 'Illinois': 'IL', 'Arkansas': 'AR',
        'New Mexico': 'NM', 'Indiana': 'IN', 'Maryland': 'MD', 'Louisiana': 'LA', 'Idaho': 'ID', 'Wyoming': 'WY',
        'Tennessee': 'TN', 'Arizona': 'AZ', 'Iowa': 'IA', 'Michigan': 'MI', 'Kansas': 'KS', 'Utah': 'UT',
        'Virginia': 'VA', 'Oregon': 'OR', 'Connecticut': 'CT', 'Montana': 'MT', 'California': 'CA',
        'Massachusetts': 'MA', 'West Virginia': 'WV', 'South Carolina': 'SC', 'New Hampshire': 'NH',
        'Wisconsin': 'WI', 'Vermont': 'VT', 'Georgia': 'GA', 'North Dakota': 'ND', 'Pennsylvania': 'PA',
        'Florida': 'FL', 'Alaska': 'AK', 'Kentucky': 'KY', 'Hawaii': 'HI', 'Nebraska': 'NE', 'Missouri': 'MO',
        'Ohio': 'OH', 'Alabama': 'AL', 'New York': 'NY', 'South Dakota': 'SD', 'Colorado': 'CO', 'New Jersey': 'NJ',
        'Washington': 'WA', 'North Carolina': 'NC', 'District of Columbia': 'DC', 'Texas': 'TX', 'Nevada': 'NV',
        'Maine': 'ME', 'Rhode Island': 'RI'}

    def __init__(self, suffixes=None, cities=None, streets=None, backend="default", dstk_api_base=None, logger=None, required_confidence=0.65):
        """
        suffixes, cities and streets provide a chance to use different lists than the provided lists.
        suffixes is probably good for most users, unless you have some suffixes not recognized by USPS.
        cities is a very expansive list that may lead to false positives in some cases. If you only have a few cities
        you know will show up, provide your own list for better accuracy. If you are doing addresses across the US,
        the provided list is probably better.
        streets can be used to limit the list of possible streets the address are on. It comes blank by default and
        uses positional clues instead. If you are instead just doing a couple cities, a list of all possible streets
        will decrease incorrect street names.
        Valid backends include "default" and "dstk". If backend is dstk, it requires a dstk_api_base. Example of
        dstk_api_base would be 'http://example.com'.
        """
        self.logger = logger
        self.backend = backend
        self.dstk_api_base = dstk_api_base
        self.required_confidence = required_confidence
        if suffixes:
            self.suffixes = suffixes
        else:
            self.load_suffixes(os.path.join(cwd, "suffixes.csv"))
        if cities:
            self.cities = cities
        else:
            self.load_cities(os.path.join(cwd, "cities.csv"))
        if streets:
            self.streets = streets
        else:
            self.load_streets(os.path.join(cwd, "streets.csv"))
        if backend == "dstk":
            if dstk_api_base is None:
                raise ValueError("dstk_api_base is required for dstk backend.")
            self.dstk = dstk.DSTK({'apiBase': dstk_api_base})
        elif backend == "default":
            pass
        else:
            raise ValueError("backend must be either 'default' or 'dstk'.")

    def parse_address(self, address, line_number=-1):
        """
        Return an Address object from the given address. Passes itself to the Address constructor to use all the custom
        loaded suffixes, cities, etc.
        """
        return Address(address, self, line_number, self.logger)

    def dstk_multi_address(self, address_list):
        if self.backend != "dstk":
            raise ValueError("Only allowed for DSTK backends.")
        if self.logger: self.logger.debug("Sending {0} possible addresses to DSTK".format(len(address_list)))
        multi_address = self.dstk.street2coordinates(address_list)
        if self.logger: self.logger.debug("Received {0} addresses from DSTK".format(len(multi_address)))
        # if self.logger: self.logger.debug("End street2coords")
        addresses = []
        # if self.logger: self.logger.debug("Multi Addresses: {0}".format(multi_address))
        for address, dstk_return in multi_address.items():
            if dstk_return is None:
                # if self.logger: self.logger.debug("DSTK None return for: {0}".format(address))
                continue
            addresses.append(Address(address, self, -1, self.logger, dstk_pre_parse=dstk_return))
            if self.logger: self.logger.debug("DSTK Address Appended: {0}".format(dstk_return))
        return addresses

    def load_suffixes(self, filename):
        """
        Build the suffix dictionary. The keys will be possible long versions, and the values will be the
        accepted abbreviations. Everything should be stored using the value version, and you can search all
        by using building a set of self.suffixes.keys() and self.suffixes.values().
        """
        with open(filename, 'r') as f:
            for line in f:
                # Make sure we have key and value
                if len(line.split(',')) != 2:
                    continue
                    # Strip off newlines.
                self.suffixes[line.strip().split(',')[0]] = line.strip().split(',')[1]

    def load_cities(self, filename):
        """
        Load up all cities in lowercase for easier matching. The file should have one city per line, with no extra
        characters. This isn't strictly required, but will vastly increase the accuracy.
{'fraction_non_alphanumeric': 0.13234641006661732, 'fraction_numerical': 0.0023686158401184307, 'mean_word_length': 2.734383637368712, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1954685', 'n_tokens_mistral': 4037, 'n_tokens_neox': 3888, 'n_words': 1405}