<jupyter_start><jupyter_text>01 Data PreparationTo the pre-prepared COMPAS data we add predictions for name gender and origin made as made by NamSor API.<jupyter_code># >>> Import Libraries

print("Importing necessary libraries... ")

import openapi_client #NamSor, see https://github.com/namsor/namsor-python-sdk2
from openapi_client.rest import ApiException

import pandas as pd

print("Libraries imported.")
# >>> Import COMPAS data set

print("Importing COMPAS data set... ")

df = pd.read_csv("data/compas_for_namsor.csv")

print("Data set imported. It is has {} entries and looks like this:".format(df.shape[0]))
df.head()
# >>> Preparing for API use

# Get private API Key for NamSor API v2 (contained in txt file)
print("Getting private key... ")

key = ''

try:
    with open("key.txt", "r") as file:
        key = file.read()
    if(len(key) < 0):
        raise FileNotFoundError()
except (FileNotFoundError):
    print("Could not find private key. Please make sure you have an API key that you stored as key.txt in the root folder.")

print("Got private key.")
print("Setting up NamSor API v2 connection settings...")

# Configure API key authorization: api_key
configuration = openapi_client.Configuration()
configuration.api_key['X-API-KEY'] = key

# create an instance of the personal API class
pers_api_instance = openapi_client.PersonalApi(openapi_client.ApiClient(configuration))

print("Connection set.")
# >>> Classifying names with NamSor API

# Formatting a df of names
print('Formatting names dataframe...')

names_df = df[['entity_id', 'first', 'last']]

print('Names dataframe formatted. It looks like this: ')
print(names_df.head())
def to_first_last_name_geo_in(row) :
    ''' This function turns a tuple of values [id, first_name, last_name] into a to_first_last_name_geo_in object'''
    # https://github.com/namsor/namsor-python-sdk2/blob/master/docs/FirstLastNameGeoIn.md
    if(not row[0] or not row[1] or not row[2]):
        print("Entered invalid data to be turned into to_first_last_name_geo_in")
        return
    return openapi_client.FirstLastNameGeoIn(id=row[0], 
                                         first_name=row[1], 
                                         last_name=row[2], 
                                         country_iso2='us') # http://www.vas.com/Tnotes/Country%20Codes.htm
# Formatting a list of batches from the names df so names can be fed to the API batch-wise
print('Creating list of name-batches...')

names_stack = list() # this will be a list of name-batches generated from the df

limit = len(names_df.index)
start = 0
end = -1
batch_size = 1000 #1000 is the API limit given by NamSor

while(end < limit):
    start = end + 1
    
    try_end = start + batch_size
    if(try_end <= limit):
        end = try_end
    else:
        end = limit
    
    # each list item will fit openapi_client.BatchFirstLastNameGeoIn
    current_df_batch = names_df[start:end]
    
    # https://stackoverflow.com/questions/16476924/how-to-iterate-over-rows-in-a-dataframe-in-pandas/55557758#55557758
    list_first_last_name_geo_in = [to_first_last_name_geo_in(row) for row in current_df_batch[['entity_id', 'first', 'last']].to_numpy()]
    names_stack.append(list_first_last_name_geo_in)
    
print('List of batches created.')

print('Will need to make {} calls.'.format(len(names_stack)))
def get_batch(list_first_last_name_geo_in):
    return openapi_client.BatchFirstLastNameGeoIn(personal_names=list_first_last_name_geo_in)
def predict_gender_batch(list_first_last_name_geo_in):
    api_response = pers_api_instance.gender_geo_batch(batch_first_last_name_geo_in=batch_first_last_name_geo_in)# call api
    return api_response.personal_names
def predict_ethnicity_batch(list_first_last_name_geo_in):
    # "Output is W_NL (white, non latino), HL (hispano latino), A (asian, non latino), B_NL (black, non latino)."
    api_response = pers_api_instance.us_race_ethnicity_batch(batch_first_last_name_geo_in=batch_first_last_name_geo_in)# call api
    return api_response.personal_names
# Sending in one batch at a time and saving the result answer by answer.

print("Sending batches to the API...")

result_gender = []
result_ethnicity = []
current = 0
limit = len(names_stack)

while(current < limit): # I assume len(result_gender) == len(result_ethnicity)
    print(current)
    batch_first_last_name_geo_in = get_batch(names_stack[current])
    try:
        result_gender.extend(predict_gender_batch(batch_first_last_name_geo_in))
        result_ethnicity.extend(predict_ethnicity_batch(batch_first_last_name_geo_in))
    except ApiException as e:
        print("Exception when calling PersonalApi: {}".format(e))
        if(len(result_gender) != (batch_size * current + len(names_stack[current])) or
          len(result_ethnicity) != (batch_size * current + len(names_stack[current]))):
            print("Some names got lost when the exception at stack {} occurred. Please try again.".format(current))
            break
        if(len(result_gender) == (batch_size * current + len(names_stack[current]))):
            print("No names got lost for gender predictions. Trying again with stack size {}...".format(len(names_stack)))
        if(len(result_ethnicity) == (batch_size * current + len(names_stack[current]))):
            print("No names got lost for ethnicity predictions. Trying again with stack size {}...".format(len(names_stack)))
        current -= 1
        continue
    current += 1

print("All batches analyzed.")
print(result_gender[:5])
print(result_ethnicity[:5])
# >>> TODO: Save results to dataframe

df.reset_index(inplace=True)
df.set_index('entity_id', inplace=True)

# Convert results (list of openapi_client.models.personal_name_gendered_out.PersonalNameGenderedOut) to (list of dictionaries)
print('Filling the results into the names dataframe...')
for i in range(len(result_gender)):
    oapi_el = result_gender[i]
    current_id = int(oapi_el.id)
    df.loc[current_id, 'sex_pred'] = oapi_el.likely_gender
    df.loc[current_id, 'sex_pred_prob'] = oapi_el.probability_calibrated
    
    oapi_el = result_ethnicity[i]
    df.loc[current_id, 'race_pred'] = oapi_el.race_ethnicity
    df.loc[current_id, 'race_pred_prob'] = oapi_el.probability_calibrated

print('Dataframe completed with API results. Here are some results: {}'.format(df.head()))
# Saving results to 'names_cat.csv'
print("Saving compas dataframe with predictions for gender and ethnicity to CSV... ")
df.to_csv("data/compas_with_predictions.csv")
print("CSV saved!")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10807510303770416, 'fraction_numerical': 0.008700961685238895, 'mean_word_length': 4.2, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 20, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '503489', 'license': 'MIT', 'n_tokens_mistral': 2142, 'n_tokens_neox': 2074, 'n_words': 532, 'path': '01-Data-Preparation-02-Data-Completion.ipynb', 'repo_name': 'namsor/namsor-compass', 'size': '6551'}