<jupyter_start><jupyter_text>Mapping electrificationCode to download results from S3We need to convert certain fields in the results to a raster; these fields need to be calculated 1. Population already electrified 2030      a. Rasterize field **ElecPop**2. New population electrified by grid 2030      a. Filter data where **FinalElecCode2030** == 1.0      b. Set **NewConnections2025** = 0 where **FinalElecCode2025** == 99.0      c. Combine **NewConnections2030** with **NewConnections2025**  3. New population electrified by off grid 2030      a. Filter data where **FinalElecCode2030** != 1.0      b. Set **NewConnections2025** = 0 where **FinalElecCode2025** != **FinalElecCode2030**      c. Combine **NewConnections2030** with **NewConnections2025**<jupyter_code># delete all existing rasterized results
'''all_rasters = []
for root, dirs, files in os.walk('/media/gost/DATA1/GEP/Clusters'):
    for f in files:
        if "rasterized_gep" in f:
            all_rasters.append(os.path.join(root, f))
for f in all_rasters:
    os.remove(f)'''
resolution = 0.01 #Resolution of output raster in degrees
elecRate = 0.90   #settlement electrification rate to be considered electrified
year = 2030       #year of analysis 
cluster_folder = '/media/gost/DATA1/GEP/Clusters'
scenario_base = '/media/gost/DATA1/GEP/Scenarios'
raster_folder = '/media/gost/DATA1/GEP/rasterized_countries'
vrt_folder = '/media/gost/DATA1/GEP/GEP_VRTs'

importlib.reload(GEP)
scenario = "0_0_0_0_1_0"
scenario_folder = os.path.join(scenario_base, f'{scenario}')
# Identify countries to process
zipFiles = []
processedCountries = []
for root, folders, files in os.walk(scenario_folder):    
    processedCountries.append([f for f in folders])
    for f in files:
        if f[-4:] == ".csv":
            zipFiles.append(f[:-4])
processedCountries = [item[:4] for item in zipFiles]
zipFiles #list of country/scenarios to process
importlib.reload(GEP)
# Loop through all countries and rasterize
minPop = 100
for country in processedCountries:
    xx_so = GEP.gep_results(country[:4], 
                scenariosFolder=scenario_base,
                clustersFolder = cluster_folder,
                scenario=scenario)
    outFile = os.path.join(raster_folder, f"{country[:2]}_rasterized_gep_{resolution}_{scenario}_{year}_gt{minPop}.tif")    
    if not os.path.exists(outFile):
        tPrint(f"Processing {country}")    
        res = xx_so.join_results()
        res = res.loc[res['Pop2030'] > minPop,]
        # Identify already electrified settlements and set value to 98
        res['already_electrified'] = (res['ElecPopCalib'] / res['Pop2020']) > elecRate
        res.loc[res['already_electrified'] == 1, 'FinalElecCode%s' % year] = 98
        xx_so.rasterize_results(res, outFile, res=resolution, field='FinalElecCode%s' % year)
    else:
        tPrint(f"{country} already rasterized")    
### Run a series of GDAL commands to create No Data rasters and stack them
command_file = os.path.join(vrt_folder, f'create_vrt_{resolution}_{scenario}_{year}.sh')
with open(command_file, 'w') as vrt_file:
    vrt_file.write('#!/bin/sh \n')
    # Convert 0 in data to no data
    outFolder = raster_folder
    allImages = []
    for root, folders, files in os.walk(outFolder):
        for f in files:
            if "%s_%s_%s_gt%s.tif" % (resolution, scenario, year, minPop) in f and f.endswith(".tif"):
                allImages.append(os.path.join(root, f))

    for inFile in allImages:
        outFile = inFile.replace(".tif", ".tif")
        vrt_file.write(f'gdal_translate -of GTiff -a_nodata 0 {inFile} {outFile}')
        vrt_file.write("\n")
    vrt_file.write(f'find {raster_folder} -name "*{resolution}_{scenario}_{year}_gt{minPop}_noData.tif" > tifFiles.txt')
    vrt_file.write("\n")
    vrt_file.write(f"gdalbuildvrt -input_file_list tifFiles.txt {vrt_folder}/combined_gep_rasterized_{resolution}_{scenario}_{year}.vrt")
    vrt_file.write("\n")
subprocess.call(command_file)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12424849699398798, 'fraction_numerical': 0.028306613226452906, 'mean_word_length': 3.84, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '183457', 'license': 'CC0-1.0', 'n_tokens_mistral': 1444, 'n_tokens_neox': 1328, 'n_words': 325, 'path': 'GEP_analysis/GEP_rasterize_results.ipynb', 'repo_name': 'global-electrification-platform/gep_results_analysis', 'size': '3992'}