<jupyter_start><jupyter_text>Select variable<jupyter_code>#Variable 
#var="tg_mean"
#var ="tn_mean"
#var ="tx_mean"
#var ="solidprcptot"
var ="prcptot"
#var ="dlyfrzthw"
#var ="growing_season_length"
#var ="DJC"


files85 = glob.glob(folder+"*"+"rcp85_"+var+"_annual.nc")
files45 = glob.glob(folder+"*"+"rcp45_"+var+"_annual.nc")
filesobs = glob.glob(folder+"NRCAN_obs_"+var+"_annual.nc")

#ex: ACCESS1-3_rcp45_tn_mean_annual.nc
files85
files45

filesobs<jupyter_output><empty_output><jupyter_text>FOR RCP45<jupyter_code>dsEns45= ens.create_ensemble(files45)
#dsEns45<jupyter_output><empty_output><jupyter_text>Get the percentiles<jupyter_code># Calculate ensembles percentiles on regional means
perctiles45 = ens.ensemble_percentiles(dsEns45)
perctiles45
dsEns45 =0<jupyter_output><empty_output><jupyter_text>Convert in Dataframe<jupyter_code>dfper145 = perctiles45.to_dataframe().dropna()
dfper45 = perctiles45.drop('realization').to_dataframe().dropna()
dfmean45 = dfper45.reset_index()
dfper45 = 0
#dfmean45
dfmean45.to_feather("/home/mlopez/EXEC/Processed data to clip with regions/rcp45"+var+"_forTS.feather")
dfmean45 = pd.read_feather("/home/mlopez/EXEC/Processed data to clip with regions/rcp45"+var+"_forTS.feather")<jupyter_output><empty_output><jupyter_text>FOR RCP85<jupyter_code>dsEns85= ens.create_ensemble(files85)
#dsEns85<jupyter_output><empty_output><jupyter_text>Get the percentiles<jupyter_code># Calculate ensembles percentiles on regional means
perctiles85 = ens.ensemble_percentiles(dsEns85)
perctiles85
dsEns85 =0<jupyter_output><empty_output><jupyter_text>Convert in Dataframe<jupyter_code>dfper185 = perctiles85.to_dataframe().dropna()
dfper85 = perctiles85.drop('realization').to_dataframe().dropna()
dfmean85 = dfper85.reset_index()
dfper85 = 0
#dfmean85
dfmean85.to_feather("/home/mlopez/EXEC/Processed data to clip with regions/rcp85"+var+"_forTS.feather")
dfmean85 = pd.read_feather("/home/mlopez/EXEC/Processed data to clip with regions/rcp85"+var+"_forTS.feather")<jupyter_output><empty_output><jupyter_text>OBSERVED DATA<jupyter_code>dsOBs= xr.open_mfdataset(filesobs)
dsOBs<jupyter_output><empty_output><jupyter_text>Convert in Dataframe<jupyter_code>dfperObs = dsOBs.to_dataframe().dropna()
dfmeanObs = dfperObs.reset_index()
dfperObs = 0
dfObs = dfmeanObs.rename(columns={var: var+'_Obs'})
dfObs
dfObs.to_feather("/home/mlopez/EXEC/Processed data to clip with regions/rcpObs"+var+"_forTS.feather")
dfObs = pd.read_feather("/home/mlopez/EXEC/Processed data to clip with regions/rcpObs"+var+"_forTS.feather")<jupyter_output><empty_output><jupyter_text>Merge 2 rcp dataframes<jupyter_code>df2m =  pd.merge(dfmean45, dfmean85, on=["lat","lon", "time"], suffixes=('_45', '_85'))
df2m<jupyter_output><empty_output><jupyter_text>Cut by region for rcp45, 85 and Obs  Get the lat lon regions<jupyter_code>regions_folder = "/home/mlopez/EXEC/Grids-polygons-regions/"
files_regions = glob.glob(regions_folder+"*"+"lat_lon.feather")
files_regions
listdfreg=[]
for region in files_regions:
    df = pd.read_feather(region)
    listdfreg.append(df)<jupyter_output><empty_output><jupyter_text>Create DF by region<jupyter_code>short_dict = {"DDE_STF_20K_REG_FOR_VUE_S": ("NM_REG_FOR", "RF"), 
            "DDE_STF_20K_UA_PER_VUE_S": ("PER_NO_UA", "UA"), 
              "DOM_BIO": ("NOM", "DB"), 
              "REG_ECO": ("NOM", "RE"), 
              "SDOM_BIO": ("NOM", "SDB"), 
              "Secteurs_Operations_Regionales": ("D_GENERAL", "SOR"), 
              "SREG_ECO": ("NOM", "SRE"),  
              "territoire_guide": ("TER_GUIDE", "TG") 
              }

for region, (name, short) in short_dict.items():
    print(region, name, short)
def replace_all(text, dic):
    for i, j in dic.items():
        text = text.replace(i, j)
    return text
d = { 'BAS-SAINT-LAURENT':"Bas-Saint-Laurent", 
     'SAGUENAY -LAC-SAINT-JEAN': "Saguenay -Lac-Saint-Jean",
     'CAPITALE-NATIONALE-CHAUDIÈRE-APPALACHES':"Capitale-Nationale-Chaudiere-Appalaches",
     'MAURICIE-CENTRE-DU-QUÉBEC':'Mauricie-Centre-du-Quebec','OUTAOUAIS':'Outaouais', 
     'ABITIBI-TEMISCAMINGUE':'Abitibi-Temiscamingue', 'COTE-NORD':'Cote-Nord',
     'NORD-DU-QUEBEC':'Nord-Du-Quebec', 'GASPESIE-ILES-DE-LA-MADELEINE':'Gaspesie-Iles-De-La-Madeleine', 
     'LANAUDIERE':'Lanaudiere','LAURENTIDES':"Laurentides", 
     "ESTRIE-MONTÉRÉGIE-LAVAL-MONTRÉAL":"Estrie-Monteregie-Laval-Montreal", 
     "é": "e", "É": "E", "à": "a", "è": "e", "Î": "i", "È": "E", "ô" : "o", "Ç":"C", "ç":"c",
       }<jupyter_output><empty_output><jupyter_text>Cutting rcp45 and 85 by region and getting df with 4th degree polynomial<jupyter_code>def polysmooth(years, y, degree):
    coef = np.polyfit(years, y, degree)
    return np.polyval(coef,years)
#dfmeanObs = dfdataregObs.groupby(["time", name]).mean() 

#dfmeanObs
for region, (name, short) in short_dict.items():
    print("------------------------------------------------------------------")
    print ("1/7... Cutting by region")
    print(region, name)
    dfregion = pd.read_feather("/home/mlopez/EXEC/Grids-polygons-regions/"+region+"lat_lon.feather")
    print ("2/7... Merging region with data")
    dfdatareg45 = pd.merge(dfregion, df2m, on=["lat","lon"])
    dfdataregObs = pd.merge(dfregion, dfObs, on=["lat","lon"])
    print ("3/7... Get mean by year for every sub-region")
    dfmeanUA45 = dfdatareg45
    dfmean45UA = dfmeanUA45.groupby(["time", name]).mean()
    dfmean45UA2 = dfmean45UA.reset_index()
    dfmeanObs = dfdataregObs.groupby(["time", name]).mean() 
    dfmeanObs2 = dfmeanObs.reset_index()
    print("4/7... Merging 3 cases")
    df3m = pd.merge(dfmean45UA2, dfmeanObs2, how='left', on= ["time", name] )
    print ("5/7... Split by subregion")
    df3ma = df3m[['time', name, var+'_p10_45', var+'_p50_45', var+'_p90_45',
                               var+'_p10_85', var+'_p50_85', var+'_p90_85', var+'_Obs']]
    if (var == "tg_mean" or var== "tn_mean" or var=="tx_mean"):
        df3ma[var+"_Obs"] = df3ma[var+"_Obs"]- 273.15
    if (var == "solidprcptot"):
        df3ma[var+"_Obs"] = df3ma[var+"_Obs"]/10
    df3ma[var+"_Obs"] = round(df3ma[var+"_Obs"], 2)
    for name2,dfsr in df3ma.groupby(name):
        dfsrc = dfsr.copy()
        print ("6/7... 4th Polynomial - rcp45,85")
        years = dfsrc.time.dt.year
        dfsrc["time"] = years#.values
        for p in ('_p10_45', '_p50_45', '_p90_45', '_p10_85', '_p50_85', '_p90_85'):
            if (var == "tg_mean" or var== "tn_mean" or var=="tx_mean"):
                dfsrc[var+p] = polysmooth(years,dfsrc[var+p],4) - 273.15
            if (var == "solidprcptot"):
                dfsrc[var+p] = polysmooth(years,dfsrc[var+p],4)/10
            if (var == "prcptot" or var == "dlyfrzthw" or var =="growing_season_length" or var =="DJC"):
                dfsrc[var+p] = polysmooth(years,dfsrc[var+p],4)
            dfsrc[var+p] = round(dfsrc[var+p], 2) 
        print(" -----------------------7/7 Creating CSV file for sub-region " +name2)
        dfsrc.to_csv(replace_all(name2,d)+"_"+var+"_annual.csv")<jupyter_output>------------------------------------------------------------------
1/7... Cutting by region
DDE_STF_20K_REG_FOR_VUE_S NM_REG_FOR
2/7... Merging region with data
3/7... Get mean by year for every sub-region
{'fraction_non_alphanumeric': 0.18332413031474323, 'fraction_numerical': 0.0353395913859746, 'mean_word_length': 4.25, 'pattern_counts': {'":': 17, '<': 52, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '257967', 'license': 'Apache-2.0', 'n_tokens_mistral': 3088, 'n_tokens_neox': 2874, 'n_words': 453, 'path': 'Data Processing Jupyter Notebooks/DF for Figure in R 8 variables  annual all regions.ipynb', 'repo_name': 'Ouranosinc/PortraitsClimMFFP', 'size': '7244'}