<jupyter_start><jupyter_text>Process and download GEP data<jupyter_code>import os, sys, logging, importlib, math, time, boto3, shutil
import rasterio, affine

import geopandas as gpd
import pandas as pd

from zipfile import ZipFile
from affine import Affine
from rasterio import features
from rasterio.mask import mask
from rasterio.features import rasterize

import GEP
from GEP import gep_results

'''prints the time along with the message'''
def tPrint(s):
    print("%s\t%s" % (time.strftime("%H:%M:%S"), s))
scenario = "0_0_0_0_1_0" 
base_folder = '/media/gost/DATA1/GEP'
scenario_base_folder =  os.path.join(base_folder, 'Scenarios')
scenario_folder = os.path.join(scenario_base_folder, scenario)
summaries_folder = os.path.join(base_folder, 'Scenario_Summaries')
zip_folder =       os.path.join(scenario_base_folder, 'zipfiles')<jupyter_output><empty_output><jupyter_text>Download results from S3<jupyter_code>s3_bucket = "wbg-geography01"
prefix = 'GEP/models-2021/'

s3_client = boto3.client('s3')
s3 = boto3.resource('s3')
bucket = s3.Bucket(s3_bucket)

for obj in bucket.objects.filter(Prefix=prefix):
    key = obj.key    
    if 'scenarios-summaries' in key:
        print(key)
        out_file = os.path.join(summaries_folder, os.path.basename(key))
        s3_client.download_file(s3_bucket, key, out_file)
    
    if 'scenarios-results' in key:
        print(key)
        out_file = os.path.join(scenario_folder, os.path.basename(key))
        s3_client.download_file(s3_bucket, key, out_file)<jupyter_output><empty_output><jupyter_text>Extract scenario files from zipfiles<jupyter_code>importlib.reload(GEP)
   
for zFile in os.listdir(zip_folder):
    cur_zip = os.path.join(zip_folder, zFile)
    country = "-".join(zFile.split("-")[:2])  
    out_scenario_file = os.path.join(scenario_folder, f'{country}-{scenario}.csv')
    if not os.path.exists(out_scenario_file):
        if country in ['fm-2','sb-2','so-2','vu-2']:
            spl = scenario.split("_")
            new_scenario = "_".join([spl[0],spl[1],spl[3]])
            cur_res = GEP.gep_results(country, scenariosFolder=scenario_base_folder, scenario=scenario)
            out_file = cur_res.extract_sample_scenario(new_scenario, scenario_folder)
            os.rename(out_file, out_scenario_file)        
        else:
            cur_res = GEP.gep_results(country, scenariosFolder=scenario_base_folder, scenario=scenario)
            cur_res.extract_sample_scenario(scenario, scenario_folder)
        print(country)<jupyter_output>sb-2
so-2
vu-2<jupyter_text>Connect scenario data to geospatial data<jupyter_code>cluster_folder = '/media/gost/DATA1/GEP/Clusters'
scenario_base = '/media/gost/DATA1/GEP/Scenarios'
vrt_folder = '/media/gost/DATA1/GEP/GEP_VRTs'
scenario = '0_0_0_0_1_0'
scenario_folder = os.path.join(scenario_base, scenario)
countries = [x[:4] for x in os.listdir(scenario_folder)]
countries   
importlib.reload(GEP)

for country in countries:
    tPrint(country)
    out_file = '/media/gost/DATA1/GEP/Country_Scenarios/%s_electrifrication_status.shp' % country
    if not os.path.exists(out_file):
        xx_so = GEP.gep_results(country, scenario=scenario)
        res = xx_so.join_results()
        sel_res = res.loc[:,['FinalElecCode2018','Pop','geometry']]
        sel_res.to_file(out_file)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12180451127819548, 'fraction_numerical': 0.0162406015037594, 'mean_word_length': 4.443535188216039, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '641853', 'license': 'CC0-1.0', 'n_tokens_mistral': 1236, 'n_tokens_neox': 1221, 'n_words': 206, 'path': 'GEP_analysis/GEP_Extract_Geog_Data.ipynb', 'repo_name': 'global-electrification-platform/gep_results_analysis', 'size': '3325'}