<jupyter_start><jupyter_text>EVALUATION OF COUNTRY INFERENCE METHODEvaluate how well works the method developed by checking accuracy with respect to ground truth.  Step necessary for ACII datasets, as ground truth is being used to check accuracy<jupyter_code>def import_df(route, csv, col_drop, col_modify, cols_test):
    
    df = pd.read_csv(route + csv)
    df = df.drop(columns = [col_drop])
    df[col_modify] = df[col_modify].str.replace('United States of America','United States')
    df_test = df[cols_test]
    
    return df, df_test
ACII = True

if ACII == True:
    route = '' #specify where the ACII conference datasets are saved
    col_to_drop = 'Unnamed: 0'
    col_to_modify ='CountryAffilitation'
    cols_for_testing = ['Name', 'Affiliation']


    df_2019, df_2019_test = import_df(route,'/all_2019.csv', col_to_drop, col_to_modify, cols_for_testing)
    df_2017, df_2017_test = import_df(route,'/all_2017.csv', col_to_drop, col_to_modify, cols_for_testing)
    df_2015, df_2015_test = import_df(route,'/all_2015.csv', col_to_drop, col_to_modify, cols_for_testing)
    df_2013, df_2013_test = import_df(route,'/all_2013.csv', col_to_drop, col_to_modify, cols_for_testing)
    df_2011, df_2011_test = import_df(route,'/all_2011.csv', col_to_drop, col_to_modify, cols_for_testing)
    df_2009, df_2009_test = import_df(route,'/all_2009.csv', col_to_drop, col_to_modify, cols_for_testing)
    df_2007, df_2007_test = import_df(route,'/all_2007.csv', col_to_drop, col_to_modify, cols_for_testing)
    df_2005, df_2005_test = import_df(route,'/all_2005.csv', col_to_drop, col_to_modify, cols_for_testing)
df_2015[['Affiliation', 'CountryAffilitation', 'Code']].head() # Name column is not shown due to anonymity issues<jupyter_output><empty_output><jupyter_text>Substitute some countries to normalize the data<jupyter_code>def sustitute_country_names(df, col):
    
    df[col] = df[col].str.replace('UK','United Kingdom')
    df[col] = df[col].str.replace('USA','United States')
    df[col] = df[col].str.replace('US','United States')
    df[col] = df[col].str.replace('Northern Ireland','United Kingdom')
    df[col] = df[col].str.replace('N.Ireland','United Kingdom')
    df[col] = df[col].str.replace('South Korea', 'Korea')
    
    return df<jupyter_output><empty_output><jupyter_text>3.1(Pre-processing) Import University-Country datasetAs some Affiliations do not include the country, if we first substitute all universities in the imported dataframe by the corresponding Country, we will have left values with missing countries afterwards<jupyter_code>route = '' #route of univerisity datasets

df_universities = pd.read_csv(route + '/university_dataset4.csv')
df_universities = df_universities[['institution', 'country']]
df_universities[15:20]<jupyter_output><empty_output><jupyter_text>Transform Universities dataset to dictionary in order to do the match with Affiliation Dataset<jupyter_code>def create_dict_from_df(df, cols):
    
    df = sustitute_country_names(df, cols[1])
    dict_ = dict(zip(df[cols[0]], df[cols[1]]))
    
    return dict_
universities_dict = create_dict_from_df(df_universities, ['institution', 'country'])<jupyter_output><empty_output><jupyter_text>Substitute universities values in Affiliation Dataset by the corresponding country<jupyter_code>def sustitute_university_by_country(df, dict_, col):
    
    for university, country in dict_.items():
        df[col] = df[col].str.replace(str(university), str(country))
        
    return df
def create_CountryAffiliation_column(df, orig_col, aff_col, countryaff_col):
    
    df[orig_col] = df[aff_col]
    df = sustitute_country_names(df, aff_col)
    #df = sustitute_country_names(df, orig_col)
    df = sustitute_university_by_country(df, universities_dict, aff_col)

    df[countryaff_col] = df[aff_col]
    
    return df<jupyter_output><empty_output><jupyter_text>ACII<jupyter_code>if ACII == True:
    df_2019_test = create_CountryAffiliation_column(df_2019_test, 'Affiliation_orig', 'Affiliation', 'CountryAffilitation')
    df_2017_test = create_CountryAffiliation_column(df_2017_test, 'Affiliation_orig', 'Affiliation', 'CountryAffilitation')
    df_2015_test = create_CountryAffiliation_column(df_2015_test, 'Affiliation_orig', 'Affiliation', 'CountryAffilitation')
    df_2013_test = create_CountryAffiliation_column(df_2013_test, 'Affiliation_orig', 'Affiliation', 'CountryAffilitation')
    df_2011_test = create_CountryAffiliation_column(df_2011_test, 'Affiliation_orig', 'Affiliation', 'CountryAffilitation')
    df_2009_test = create_CountryAffiliation_column(df_2009_test, 'Affiliation_orig', 'Affiliation', 'CountryAffilitation')
    df_2007_test = create_CountryAffiliation_column(df_2007_test, 'Affiliation_orig', 'Affiliation', 'CountryAffilitation')
    df_2005_test = create_CountryAffiliation_column(df_2005_test, 'Affiliation_orig', 'Affiliation', 'CountryAffilitation')
df_2015_test[['Affiliation', 'Affiliation_orig', 'CountryAffilitation']].head()<jupyter_output><empty_output><jupyter_text>The previous dataset shows:- **Affiliation** : takes the universities in the universities dictionary and substitute them by the corresponding country. For instance, in the first case University of Cambrigde substituted by United Kingdom.- **Affiliation_orig** : original affiliation column- **CountryAffiliation** : for now it does not mean anything, but it will contain the corresponding country to the affiliations.  3.3 Implementing library pycountryWhen there are two possible countries the last one is taken.<jupyter_code>import pycountry
import numpy as np

def country_inference(df):

    countries = {}
    for affiliation in df.Affiliation:
        if affiliation!= None:

            #im also saving the values where there is no country inferred
            for country in pycountry.countries:
                text = affiliation

                # to avoid null values
                if text == text:

                    if country.name in text:
                        name = country.name
                        countries[affiliation] = name
                        
                        
    # Replace original values with country
    df.CountryAffilitation = df.CountryAffilitation.replace(countries)

    return df
    
def create_final_df(df, col, col_to_substitute):
    
    df = country_inference(df)
    df = df.drop([col], axis=1)
    df = df.rename(columns={col_to_substitute: col})
    
    return df<jupyter_output><empty_output><jupyter_text>Pycountry library has a problem when reading Taiwan and Korea, manually infer this country.<jupyter_code>def taiwan_korea_inference(df):
    for i in df.index:
        if 'Taiwan' in df['CountryAffilitation'].iloc[i]:
            df.at[i, 'CountryAffilitation'] = 'Taiwan'
            
    for i in df.index:
        if 'Korea' in df['CountryAffilitation'].iloc[i]:
            df.at[i, 'CountryAffilitation'] = 'Korea'
            
    return df<jupyter_output><empty_output><jupyter_text>ACII<jupyter_code>if ACII == True:
    df_2019_test = create_final_df(df_2019_test, 'Affiliation', 'Affiliation_orig')
    df_2017_test = create_final_df(df_2017_test, 'Affiliation', 'Affiliation_orig')
    df_2015_test = create_final_df(df_2015_test, 'Affiliation', 'Affiliation_orig')
    df_2013_test = create_final_df(df_2013_test, 'Affiliation', 'Affiliation_orig')
    df_2011_test = create_final_df(df_2011_test, 'Affiliation', 'Affiliation_orig')
    df_2009_test = create_final_df(df_2009_test, 'Affiliation', 'Affiliation_orig')
    df_2007_test = create_final_df(df_2007_test, 'Affiliation', 'Affiliation_orig')
    df_2005_test = create_final_df(df_2005_test, 'Affiliation', 'Affiliation_orig')

    df_2019_test.head()
if ACII == True:
    df_2019_test = taiwan_korea_inference(df_2019_test)
    df_2017_test = taiwan_korea_inference(df_2017_test)
    df_2015_test = taiwan_korea_inference(df_2015_test)
    df_2013_test = taiwan_korea_inference(df_2013_test)
    df_2011_test = taiwan_korea_inference(df_2011_test)
    df_2009_test = taiwan_korea_inference(df_2009_test)
    df_2007_test = taiwan_korea_inference(df_2007_test)
    df_2005_test = taiwan_korea_inference(df_2005_test)<jupyter_output><empty_output><jupyter_text>Rows with no AffiliationCountry value<jupyter_code>def country_affiliation_missing(df):
    
    no_countryaffiliation = df[(df.Affiliation == df.CountryAffilitation) | (df.CountryAffilitation.isna())] 
    print('There are', len(no_countryaffiliation),'rows out of', len(df),'with missing AffiliationCountry value')
    
    for i in df.index:
        if df.Affiliation.iloc[i] == df.CountryAffilitation.iloc[i]:
            df.at[i, 'CountryAffilitation'] = np.nan
    
    return df, no_countryaffiliation<jupyter_output><empty_output><jupyter_text>ACII<jupyter_code>if ACII == True:
    df_2019_test, df_2019_test_nocountry = country_affiliation_missing(df_2019_test)
    df_2017_test, df_2017_test_nocountry = country_affiliation_missing(df_2017_test)
    df_2015_test, df_2015_test_nocountry = country_affiliation_missing(df_2015_test)
    df_2013_test, df_2013_test_nocountry = country_affiliation_missing(df_2013_test)
    df_2011_test, df_2011_test_nocountry = country_affiliation_missing(df_2011_test)
    df_2009_test, df_2009_test_nocountry = country_affiliation_missing(df_2009_test)
    df_2007_test, df_2007_test_nocountry = country_affiliation_missing(df_2007_test)
    df_2005_test, df_2005_test_nocountry = country_affiliation_missing(df_2005_test)
# Examples of affiliations where the country could not be inferred
df_2017_test_nocountry[0:len(df_2015_test_nocountry)][0:10]['Affiliation']<jupyter_output><empty_output><jupyter_text>TEST : manually inferring location in order to see how my implementation works        The ACII Conference was already labelled and revised manually, therefore the accuracy of the new method develped was checked using this as ground truth.<jupyter_code>def wrong_country(df):
    for i in df.index:
        if df.CountryAffilitation.iloc[i] != df.CountryAffiliation_infered.iloc[i]:
            df.at[i, 'WrongCountry'] = 'Yes'
        else:
            df.at[i,'WrongCountry'] = 'No'
    return df<jupyter_output><empty_output><jupyter_text>ACII<jupyter_code>if ACII == True:
    df_2019['CountryAffiliation_infered']= df_2019_test.CountryAffilitation
    df_2017['CountryAffiliation_infered']= df_2017_test.CountryAffilitation
    df_2015['CountryAffiliation_infered']= df_2015_test.CountryAffilitation
    df_2013['CountryAffiliation_infered']= df_2013_test.CountryAffilitation
    df_2011['CountryAffiliation_infered']= df_2011_test.CountryAffilitation
    df_2009['CountryAffiliation_infered']= df_2009_test.CountryAffilitation
    df_2007['CountryAffiliation_infered']= df_2007_test.CountryAffilitation
    df_2005['CountryAffiliation_infered']= df_2005_test.CountryAffilitation
 if ACII == True:
    count_2019 = (len(df_2019[df_2019.CountryAffilitation != df_2019.CountryAffiliation_infered])/len(df_2019))*100
    count_2017 = (len(df_2017[df_2017.CountryAffilitation != df_2017.CountryAffiliation_infered])/len(df_2017))*100
    count_2015 = (len(df_2015[df_2015.CountryAffilitation != df_2015.CountryAffiliation_infered])/len(df_2015))*100
    count_2013 = (len(df_2013[df_2013.CountryAffilitation != df_2013.CountryAffiliation_infered])/len(df_2013))*100
    count_2011 = (len(df_2011[df_2011.CountryAffilitation != df_2011.CountryAffiliation_infered])/len(df_2011))*100
    count_2009 = (len(df_2009[df_2009.CountryAffilitation != df_2009.CountryAffiliation_infered])/len(df_2009))*100
    count_2007 = (len(df_2007[df_2007.CountryAffilitation != df_2007.CountryAffiliation_infered])/len(df_2007))*100
    count_2005 = (len(df_2005[df_2005.CountryAffilitation != df_2005.CountryAffiliation_infered])/len(df_2005))*100
    
    count_not_2019 = (len(df_2019_test_nocountry)/len(df_2019_test))*100
    count_not_2017 = (len(df_2017_test_nocountry)/len(df_2017_test))*100
    count_not_2015 = (len(df_2015_test_nocountry)/len(df_2015_test))*100
    count_not_2013 = (len(df_2013_test_nocountry)/len(df_2013_test))*100
    count_not_2011 = (len(df_2011_test_nocountry)/len(df_2011_test))*100
    count_not_2009 = (len(df_2009_test_nocountry)/len(df_2009_test))*100
    count_not_2007 = (len(df_2007_test_nocountry)/len(df_2007_test))*100
    count_not_2005 = (len(df_2005_test_nocountry)/len(df_2005_test))*100
    
    acc_2019 = (1- (count_2019/100))*100
    acc_2017 = (1- (count_2017/100))*100
    acc_2015 = (1- (count_2015/100))*100
    acc_2013 = (1- (count_2013/100))*100
    acc_2011 = (1- (count_2011/100))*100
    acc_2009 = (1- (count_2009/100))*100
    acc_2007 = (1- (count_2007/100))*100
    acc_2005 = (1- (count_2005/100))*100
    acc = [acc_2019, acc_2017, acc_2015, acc_2013, acc_2011, acc_2009, acc_2007, acc_2005]
if ACII == True:
    data = {'2019':[count_2019, count_not_2019, acc_2019],
            '2017':[count_2017, count_not_2017, acc_2017],
            '2015':[count_2015, count_not_2015, acc_2015],
            '2013':[count_2013, count_not_2013, acc_2013],
            '2011':[count_2011, count_not_2011, acc_2011],
            '2009':[count_2009, count_not_2009, acc_2009],
            '2007':[count_2007, count_not_2007, acc_2007],
            '2005':[count_2005, count_not_2005, acc_2005]}
    
    df_results = pd.DataFrame(data, index =[
                                    '% of countries NOT coinciding with ground truth',
        '% of countries NOT inferred ',
                                    'ACCURACY'])
    
    df_results = df_results.round(2)
    
    for i in df_results.columns:
        df_results[i] =  df_results[i].astype(str) + ' %'
    
    mean_acc = np.mean(acc).round(3)
        
    print('MEAN ACCURACY OF COUNTRY INFERENCE:', mean_acc, '%')
df_results.head()
if ACII == True:
    df_2019 = wrong_country(df_2019)
    df_2017 = wrong_country(df_2017)
    df_2015 = wrong_country(df_2015)
    df_2013 = wrong_country(df_2013)
    df_2011 = wrong_country(df_2011)
    df_2009 = wrong_country(df_2009)
    df_2007 = wrong_country(df_2007)
    df_2005 = wrong_country(df_2005)
df_2007[['Affiliation', 'CountryAffilitation', 'CountryAffiliation_infered', 'WrongCountry']][0:10]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10334666386023995, 'fraction_numerical': 0.08019364344348558, 'mean_word_length': 5.070698466780239, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '359214', 'license': 'MIT', 'n_tokens_mistral': 5724, 'n_tokens_neox': 4821, 'n_words': 851, 'path': 'evaluation/Evaluation Country Inference with Ground Truth.ipynb', 'repo_name': 'saraestevez02/AI-diversity-analysis', 'size': '14253'}