<jupyter_start><jupyter_text>Join scenario results to cluster shapefile<jupyter_code>country = 'cm-1'
xx_so = gep_results(country, 
                    scenariosFolder=f'/media/gost/DATA1/GEP/Scenarios/Scenario_{scenario}',
                    clustersFolder = '/media/gost/DATA1/GEP/Clusters',
                    scenario=scenario)
inD = xx_so.join_results()
inD.head()
inD.to_file('/media/gost/DATA1/GEP/Clusters/GEP_clusters_%s.shp' % scenario)
tempD = inD.loc[inD['Pop'] > 10,['FinalElecCode2025','FinalElecCode2030','Population','geometry']]
tempD.shape

allRes = {}
cnt = 0
for country in countries:
    if not country in allRes.keys():
        print(f'Processing {country}')
        xx_so = gep_results(country, 
                    scenariosFolder=f'/media/gost/DATA1/GEP/Scenario_{scenario}',
                    clustersFolder = '/media/gost/DATA1/GEP/Clusters',
                    scenario=scenario)
        inD = xx_so.join_results()
        # Identify rows where the electrified source changes between 2025 and 2030
        changing_source = (inD['FinalElecCode2025'] > inD['FinalElecCode2030']) & (inD['FinalElecCode2025'] != 99)
        # For those changed rows, set the adjusted 2030 value = 2025 value
        inD['adjust_2030'] = inD['FinalElecCode2030']
        inD['ElecPop2030'] = inD['Pop2030'] - inD['ElecPop']
        inD.loc[changing_source,'adjust_2030'] = inD.loc[changing_source, 'FinalElecCode2025']
        allRes[country] = [inD.loc[:,['adjust_2030','ElecPop2030','ElecPop','Pop']].groupby('adjust_2030').sum(),
                           inD.loc[:,['FinalElecCode2030','ElecPop2030','ElecPop','Pop']].groupby('FinalElecCode2030').sum()]
    else:
        print(f'Already processed {country}')
    '''cnt += 1
    if cnt > 1:
        break'''
allRes = {}
cnt = 0
for country in countries:
    if not country in allRes.keys():
        print(f'Processing {country}')
        xx_so = gep_results(country, 
                    scenariosFolder=f'/media/gost/DATA1/GEP/Scenario_{scenario}',
                    clustersFolder = '/media/gost/DATA1/GEP/Clusters',
                    scenario=scenario)
        inD = xx_so.join_results()
        # Identify rows where the electrified source changes between 2025 and 2030
        changing_source = (inD['FinalElecCode2025'] > inD['FinalElecCode2030']) & (inD['FinalElecCode2025'] != 99)
        # For those changed rows, set the adjusted 2030 value = 2025 value
        inD['adjust_2030'] = inD['FinalElecCode2030']
        inD['ElecPop2030'] = inD['Pop2030'] - inD['ElecPop']
        inD.loc[changing_source,'adjust_2030'] = inD.loc[changing_source, 'FinalElecCode2025']
        allRes[country] = [inD.loc[:,['adjust_2030','ElecPop2030','ElecPop','Pop','NewConnections2030','NewConnections2025']].groupby('adjust_2030').sum(),
                           inD.loc[:,['FinalElecCode2030','ElecPop2030','ElecPop','Pop','NewConnections2030','NewConnections2025']].groupby('FinalElecCode2030').sum()]
    else:
        print(f'Already processed {country}')
    '''cnt += 1
    if cnt > 1:
        break'''
orig = allRes.copy()
final.head()
# del(final)
allRes = orig
for key, value in allRes.items():    
    combined = value[0].join(value[1], lsuffix="_adj", rsuffix='_orig')
    combined.columns = [f'{x}_{key}' for x in combined.columns]        
    try:
        final = final.join(combined, rsuffix=key)
    except:
        final = combined
os.getcwd()
final2.to_csv("final2.csv")
importlib.reload(GEP)
final2 = final.transpose()
final2['type'] = [x.split("_")[1] for x in final2.index]
final2['pop'] = [x.split("_")[0] for x in final2.index]
final2['continent'] = [GEP.get_continent(x.split("_")[2]) for x in final2.index]
final2.groupby(['type','pop','continent']).sum()
final2.loc['ElecPop2030_orig_bi-1']
final2.continent.unique()
final2.to_csv("final2.csv", sep=";", na_rep=0)
final3 = final2.groupby(['type','pop','continent']).sum()
final3.to_csv("final3.csv", sep=";")
#del(final)
final.to_csv(f'{scenariosFolder}_adjusted2.csv')
adjusted_values = value[0]    
    adjusted_values.columns = [f'{key}_{x}' for x in adjusted_values.columns]
    orig_values = value[1]
    orig_values.columns = [f'{key}_{x}' for x in orig_values.columns]
    try:
        final_adj = final_adj.join(adjusted_values)
    
value[0]
inD.loc[inD['Pop2030'] > 100,].shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15970628728774666, 'fraction_numerical': 0.05415328132170721, 'mean_word_length': 3.554858934169279, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '286761', 'license': 'CC0-1.0', 'n_tokens_mistral': 1674, 'n_tokens_neox': 1522, 'n_words': 247, 'path': 'GEP_analysis/GEP_geographic_inspections.ipynb', 'repo_name': 'global-electrification-platform/gep_results_analysis', 'size': '4358'}