<jupyter_start><jupyter_text>clean data<jupyter_code># population data
pop_df_raw = pd.read_csv("population_data/API_SP.POP/API_SP.POP.csv")
pop_obj = list()
for index, row in pop_df_raw.iterrows():
    for elem in reversed(row):
        # select last known population
        if elem == elem:
            population = elem
            break
    if isinstance(elem, float):
        pop_obj.append({
            "country": row["Country Code"],
            "population": population
        })
pop_df = pd.DataFrame(pop_obj)
pop_df.to_csv("population_data/pop_clean.csv")

# gdp data
gdp_df_raw = pd.read_csv("population_data/API_NY.GDP/API_NY.GDP.csv")
gdp_obj = list()
for index, row in gdp_df_raw.iterrows():
    for elem in reversed(row):
        # select last known gdp
        if elem == elem:
            gdp = elem
            break
    if isinstance(elem, float):
        gdp_obj.append({
            "country": row["Country Code"],
            "gdp": gdp
        })
gdp_df = pd.DataFrame(gdp_obj)
gdp_df.to_csv("population_data/gdp_clean.csv")<jupyter_output><empty_output><jupyter_text>add population and gdp data to lcc data<jupyter_code>with open("stats/lcc_country.json", "r", encoding="utf8") as in_file:
    countries = json.load(in_file)

for alpha_2, data in countries.items():
    # if alpha_2 == "INTERNATIONAL":
    #     continue
    country = pycountry.countries.get(alpha_2=alpha_2)
    if country is None:
        continue
    alpha_3 = country.alpha_3
    population = pop_df.loc[pop_df['country'] == alpha_3]["population"]
    gdp = gdp_df.loc[gdp_df['country'] == alpha_3]["gdp"]
    if len(population) > 0:
        data["population"] = population.values[0]
    if len(gdp) > 0:
        data["gdp"] = gdp.values[0]

with open("stats/lcc_pop_gdp_country.json", "w", encoding="utf8") as out_file:
    json.dump(countries, out_file)

# dataframe and csv
country_df = pd.DataFrame.from_dict(countries, orient="index")
country_df = country_df.query("count != 0")
country_df.dropna(inplace=True, subset=["population", "gdp"], how="all")
# country_df.index.name = "country"
country_df.to_csv("stats/lcc_pop_gdp_country.csv")
country_df
country = pycountry.countries.get(alpha_2="MF")
country<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13443501563197857, 'fraction_numerical': 0.007146047342563644, 'mean_word_length': 3.5807770961145193, 'pattern_counts': {'":': 5, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '369383', 'license': 'Apache-2.0', 'n_tokens_mistral': 805, 'n_tokens_neox': 789, 'n_words': 152, 'path': 'population.ipynb', 'repo_name': 'netgroup/as-analysis', 'size': '2239'}