<commit_before>"""
Use Google Maps Geocoding API.

Assign missing Lat|Lng to business addresses in Chicago, IL.
Excludes PO Box Addresses.
"""
# !/usr/bin/env python
# -*- coding: utf8 -*-
import sys

import pandas as pd
import re
import requests
import google_api


def drop_PO_box(df, check='BOX '):
    """Filter PO Box addresses."""
    return df[~df['ADDRESS'].str.contains(check)]


def drop_outside_IL(df, check='IL'):
    """Filter addresses outside IL."""
    return df[df['STATE'].str.contains(check)]


def drop_outside_Chicago(df, check='CHICAGO'):
    """Filter addresses outside Chicago."""
    return df[df['CITY'] == check]


def strip_end_noise(str):
    """Remove junk from end of addresses."""
    return str.split("#")[0].split('&')[0].split(',')[0].split('/')[0].strip()


def end_has_numbers(inputString):
    """Remove last element in address if it contains number."""
    if len(inputString) == 0:
        return False
    else:
        return bool(re.search(r'\d', inputString.split()[-1]))


def end_is_single_char(inputString):
    """Remove last element in address if 1 element long."""
    if len(inputString) == 0:
        return False
    else:
        return bool(len(inputString.split()[-1]) <= 1)


def add_clean_street_address(df):
    """Return df with new column for CLEAN_ADDRESS to be used for API."""
    df['CLEAN_ADDRESS'] = df['ADDRESS'].apply(lambda x: strip_end_noise(x))
    df['CLEAN_ADDRESS'] = df['CLEAN_ADDRESS'].apply(lambda x:
                                                    ' '.join(x.split()[:-1])
                                                    if end_has_numbers(x)
                                                    else x)
    df['CLEAN_ADDRESS'] = df['CLEAN_ADDRESS'].apply(lambda x:
                                                    ' '.join(x.split()[:-1])
                                                    if end_is_single_char(x)
                                                    else x)
    df['CLEAN_ADDRESS'] = df['CLEAN_ADDRESS'].apply(lambda x:
                                                    x.replace('PKWY',
                                                              'PARKWAY'))
    df['CLEAN_ADDRESS'] = df['CLEAN_ADDRESS'].apply(lambda x:
                                                    x.replace('AVE RD',
                                                              'AVE'))
    df['CLEAN_ADDRESS'] = df['CLEAN_ADDRESS'].apply(lambda x:
                                                    x.replace('AVE AVE',
                                                              'AVE'))
    df = df.drop_duplicates('ADDRESS').copy()
    return df


def get_request(street, city, state, key):
    """Make API call and return request for passed params."""
    url = 'https://maps.googleapis.com/maps/api/geocode/json?address='\
          + street + ',' + city + ',' + state + '&key=' + key
    return requests.get(url)


def extract_lat_lng(r):
    """Extract and return latitue and longitude from API response."""
    lat = r.json()['results'][0]['geometry']['location']['lat']
    lng = r.json()['results'][0]['geometry']['location']['lng']
    return lat, lng


def make_address_request(street, city, state, key):
    """Make API request for passed params. Final clean of street address."""
    street = street.replace('  ', '+').replace(' ', '+')
    return get_request(street, city, state, key)


# retrieve API key from config file
key = google_api.google_api_key
# load in raw business
df = pd.read_csv(sys.stdin, low_memory=False)
# Filter addresses for only street address in Chicago, IL
latlng_df = df[pd.isnull(df['LOCATION'])].copy()
latlng_df = drop_PO_box(latlng_df)
latlng_df = drop_outside_IL(latlng_df)
latlng_df = drop_outside_Chicago(latlng_df)
# Remove trialing noise from addresses (i.e., apparment numbers, letters, etc)
latlng_df = add_clean_street_address(latlng_df)

# Set of a clean addresses to send to API for LAT | LNG
clean_addr = set(latlng_df.CLEAN_ADDRESS)

# Make dictionary to store address to lat|lng mapping
addr_to_coordinate_dict = {}
# API call to retrieve lat lng for each clean addr
for addr in clean_addr:
    # skip blank, else api wiil return chicago, il lat|lng
    if addr == '':
        continue
    street = addr
    city = 'CHICAGO'
    state = 'IL'
    # Call API
    r = make_address_request(street, city, state, key)
    # Check if over API limit
    if r.json()['status'] == 'OVER_QUERY_LIMIT':
        print 'Congrats! You and your team have exceeded the daily request'\
              'quota for Google\'s API. Try again tomorrow '\
              '\xc2\xaf\\_(\xe3\x83\x84)_/\xc2\xaf'
        break
    else:
        lat, lng = extract_lat_lng(r)
        addr_to_coordinate_dict[addr] = [lat, lng]

# Store LATITUDE and LONGITUDE in dataframe
for k, v in addr_to_coordinate_dict.iteritems():
    latlng_df.loc[latlng_df['CLEAN_ADDRESS'] == k, 'LATITUDE'] = v[0]
    latlng_df.loc[latlng_df['CLEAN_ADDRESS'] == k, 'LONGITUDE'] = v[1]

# Merge only certain columns with original df
df.loc[df.ADDRESS.isin(latlng_df.ADDRESS),
       ['LATITUDE', 'LONGITUDE']] = latlng_df[['LATITUDE', 'LONGITUDE']]

# SAVE OUTPUT
df.to_csv(sys.stdout)<commit_msg>Add comment to explain escaped ¯\_(ツ)_/¯ in error message<commit_after>"""
Use Google Maps Geocoding API.

Assign missing Lat|Lng to business addresses in Chicago, IL.
Excludes PO Box Addresses.
"""
# !/usr/bin/env python
# -*- coding: utf8 -*-
import sys

import pandas as pd
import re
import requests
import google_api


def drop_PO_box(df, check='BOX '):
    """Filter PO Box addresses."""
    return df[~df['ADDRESS'].str.contains(check)]


def drop_outside_IL(df, check='IL'):
    """Filter addresses outside IL."""
    return df[df['STATE'].str.contains(check)]


def drop_outside_Chicago(df, check='CHICAGO'):
    """Filter addresses outside Chicago."""
    return df[df['CITY'] == check]


def strip_end_noise(str):
    """Remove junk from end of addresses."""
    return str.split("#")[0].split('&')[0].split(',')[0].split('/')[0].strip()


def end_has_numbers(inputString):
    """Remove last element in address if it contains number."""
    if len(inputString) == 0:
        return False
    else:
        return bool(re.search(r'\d', inputString.split()[-1]))


def end_is_single_char(inputString):
    """Remove last element in address if 1 element long."""
    if len(inputString) == 0:
        return False
    else:
        return bool(len(inputString.split()[-1]) <= 1)


def add_clean_street_address(df):
    """Return df with new column for CLEAN_ADDRESS to be used for API."""
    df['CLEAN_ADDRESS'] = df['ADDRESS'].apply(lambda x: strip_end_noise(x))
    df['CLEAN_ADDRESS'] = df['CLEAN_ADDRESS'].apply(lambda x:
                                                    ' '.join(x.split()[:-1])
                                                    if end_has_numbers(x)
                                                    else x)
    df['CLEAN_ADDRESS'] = df['CLEAN_ADDRESS'].apply(lambda x:
                                                    ' '.join(x.split()[:-1])
                                                    if end_is_single_char(x)
                                                    else x)
    df['CLEAN_ADDRESS'] = df['CLEAN_ADDRESS'].apply(lambda x:
                                                    x.replace('PKWY',
                                                              'PARKWAY'))
    df['CLEAN_ADDRESS'] = df['CLEAN_ADDRESS'].apply(lambda x:
                                                    x.replace('AVE RD',
                                                              'AVE'))
    df['CLEAN_ADDRESS'] = df['CLEAN_ADDRESS'].apply(lambda x:
                                                    x.replace('AVE AVE',
                                                              'AVE'))
    df = df.drop_duplicates('ADDRESS').copy()
    return df


def get_request(street, city, state, key):
    """Make API call and return request for passed params."""
    url = 'https://maps.googleapis.com/maps/api/geocode/json?address='\
          + street + ',' + city + ',' + state + '&key=' + key
    return requests.get(url)


def extract_lat_lng(r):
    """Extract and return latitue and longitude from API response."""
    lat = r.json()['results'][0]['geometry']['location']['lat']
    lng = r.json()['results'][0]['geometry']['location']['lng']
    return lat, lng


def make_address_request(street, city, state, key):
    """Make API request for passed params. Final clean of street address."""
    street = street.replace('  ', '+').replace(' ', '+')
    return get_request(street, city, state, key)


# retrieve API key from config file
key = google_api.google_api_key
# load in raw business
df = pd.read_csv(sys.stdin, low_memory=False)
# Filter addresses for only street address in Chicago, IL
latlng_df = df[pd.isnull(df['LOCATION'])].copy()
latlng_df = drop_PO_box(latlng_df)
latlng_df = drop_outside_IL(latlng_df)
latlng_df = drop_outside_Chicago(latlng_df)
# Remove trialing noise from addresses (i.e., apparment numbers, letters, etc)
latlng_df = add_clean_street_address(latlng_df)

# Set of a clean addresses to send to API for LAT | LNG
clean_addr = set(latlng_df.CLEAN_ADDRESS)

# Make dictionary to store address to lat|lng mapping
addr_to_coordinate_dict = {}
# API call to retrieve lat lng for each clean addr
for addr in clean_addr:
    # skip blank, else api wiil return chicago, il lat|lng
    if addr == '':
        continue
    street = addr
    city = 'CHICAGO'
    state = 'IL'
    # Call API
    r = make_address_request(street, city, state, key)
    # Check if over API limit
    if r.json()['status'] == 'OVER_QUERY_LIMIT':
        print 'Congrats! You and your team have exceeded the daily request'\
              'quota for Google\'s API. Try again tomorrow '\
              '\xc2\xaf\\_(\xe3\x83\x84)_/\xc2\xaf' #shrug ¯\_(ツ)_/¯
        break
    else:
        lat, lng = extract_lat_lng(r)
        addr_to_coordinate_dict[addr] = [lat, lng]

# Store LATITUDE and LONGITUDE in dataframe
for k, v in addr_to_coordinate_dict.iteritems():
    latlng_df.loc[latlng_df['CLEAN_ADDRESS'] == k, 'LATITUDE'] = v[0]
    latlng_df.loc[latlng_df['CLEAN_ADDRESS'] == k, 'LONGITUDE'] = v[1]

# Merge only certain columns with original df
df.loc[df.ADDRESS.isin(latlng_df.ADDRESS),
       ['LATITUDE', 'LONGITUDE']] = latlng_df[['LATITUDE', 'LONGITUDE']]

# SAVE OUTPUT
df.to_csv(sys.stdout)
{'fraction_non_alphanumeric': 0.1337738197835871, 'fraction_numerical': 0.004596380350474002, 'mean_word_length': 2.3271742593182543, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2328481', 'n_tokens_mistral': 3223, 'n_tokens_neox': 3121, 'n_words': 825}