<jupyter_start><jupyter_text>Select variable<jupyter_code>#Variable 
var="tg_mean"
#var ="tn_mean"
#var ="tx_mean"
#var ="solidprcptot"
#var ="prcptot"
#var ="dlyfrzthw"
#var ="DJC"


files85 = glob.glob(folder+"*"+"rcp85_"+var+"_seasonal.nc")
files45 = glob.glob(folder+"*"+"rcp45_"+var+"_seasonal.nc")
filesobs = glob.glob(folder+"NRCAN_obs_"+var+"_seasonal.nc")

#ex: ACCESS1-3_rcp45_tn_mean_annual.nc
files85
files45

filesobs<jupyter_output><empty_output><jupyter_text>FOR RCP45<jupyter_code>dsEns45= ens.create_ensemble(files45)
dsEns45.time.dt.season[0:4]
# Seasonal example 30 y means
listds45 = []
i=0
for s in dsEns45.time.dt.season[0:4]:
    print (i)
    tmp1 = dsEns45.sel(time=(dsEns45.time.dt.season==s))
    print(tmp1)
    i = i+1
    listds45.append(tmp1)
i = 0
df45list = []
for ds in listds45:
    print ("Dataset number: ", i)
    perc45 = ens.ensemble_percentiles(ds)
    print ("Percentiles Obteined")
    df45 = perc45.to_dataframe()
    df452 = df45.reset_index()
    df45=0
    df453 = df452.loc[df452["realization"] == 0].dropna()
    df452 = 0
    print ("Converted to DF")
    dfp452 =  df453.reset_index()
    df453 =0
    df45C = dfp452.copy()
    dfp452=0
    df45C[var+"_p10"] = round(df45C[var+"_p10"],2)
    df45C[var+"_p50"] = round(df45C[var+"_p50"],2)
    df45C[var+"_p90"] = round(df45C[var+"_p90"],2)
    print ("Transformed in Celsiuds and rounded")
    Region1i45 = df45C.drop(columns=["realization"])
    df45C=0
    if i== 0: season = "winter" 
    elif i == 1: season = "spring"
    elif i == 2: season = "summer"
    elif i == 3: season = "fall"
    Region1i45["season"] = season
    df45list.append(Region1i45)
    print ("DF added to the list")
    i = i + 1
    
df45 = pd.concat(df45list)
df45.reset_index().to_feather("/home/mlopez/EXEC/Processed data to clip with regions/df45"+var+"_forTS_seasonal.feather")
df45 = pd.read_feather("/home/mlopez/EXEC/Processed data to clip with regions/df45"+var+"_forTS_seasonal.feather")<jupyter_output><empty_output><jupyter_text>FOR RCP85<jupyter_code>dsEns85= ens.create_ensemble(files85)
dsEns85.time.dt.season[0:4]
# Seasonal example 30 y means
listds85 = []
i=0
for s in dsEns85.time.dt.season[0:4]:
    print (i)
    tmp1 = dsEns85.sel(time=(dsEns85.time.dt.season==s))
    print(tmp1)
    i = i+1
    listds85.append(tmp1)
i = 0
df85list = []
for ds in listds85:
    print ("Dataset number: ", i)
    perc85 = ens.ensemble_percentiles(ds)
    print ("Percentiles Obteined")
    df85 = perc85.to_dataframe()
    df852 = df85.reset_index()
    df85=0
    df853 = df852.loc[df852["realization"] == 0].dropna()
    df852=0
    print ("Converted to DF")
    dfp852 =  df853.reset_index()
    df853=0
    df85C = dfp852.copy()
    df85C[var+"_p10"] = round(df85C[var+"_p10"],2)
    df85C[var+"_p50"] = round(df85C[var+"_p50"],2)
    df85C[var+"_p90"] = round(df85C[var+"_p90"],2)
    print ("Transformed in Celsiuds and rounded")
    Region1i85 = df85C.drop(columns=["realization"])
    df85C=0
    if i== 0: season = "winter" 
    elif i == 1: season = "spring"
    elif i == 2: season = "summer"
    elif i == 3: season = "fall"
    Region1i85["season"] = season
    df85list.append(Region1i85)
    print ("DF added to the list")
    i = i + 1
    
df85 = pd.concat(df85list)
df85.reset_index().to_feather("/home/mlopez/EXEC/Processed data to clip with regions/df85"+var+"_forTS_seasonal.feather")
df85 = pd.read_feather("/home/mlopez/EXEC/Processed data to clip with regions/df85"+var+"_forTS_seasonal.feather")<jupyter_output><empty_output><jupyter_text>OBSERVED DATA<jupyter_code>dsOBs= xr.open_mfdataset(filesobs)
dsOBs<jupyter_output><empty_output><jupyter_text>Convert in Dataframe<jupyter_code>dfperObs = dsOBs.to_dataframe().dropna()
dfmeanObs = dfperObs.reset_index()
dfperObs = 0
dfObs = dfmeanObs.rename(columns={var: var+'_Obs'})
dfmeanObs=0
season_dict = {12:"winter", 3:"spring", 6:"summer", 9:"fall" }

dfObs["season"]= dfObs.time.dt.month.map(lambda m: season_dict[m])
dfObs
dfObs.to_feather("/home/mlopez/EXEC/Processed data to clip with regions/rcpObs"+var+"_forTS.feather")
dfObs = pd.read_feather("/home/mlopez/EXEC/Processed data to clip with regions/rcpObs"+var+"_forTS.feather")<jupyter_output><empty_output><jupyter_text>Merge 2 rcp dataframes<jupyter_code>df2m =  pd.merge(df45, df85, on=["lat","lon", "time", "season"], suffixes=('_45', '_85'))
df45 = 0
df85 = 0
df2m
df2m.to_feather("/home/mlopez/EXEC/Processed data to clip with regions/df2m"+var+"_forTS_seasonal.feather")
df2m = pd.read_feather("/home/mlopez/EXEC/Processed data to clip with regions/df2m"+var+"_forTS_seasonal.feather")<jupyter_output><empty_output><jupyter_text>Cut by region for rcp45, 85 and Obs  Get the lat lon regions<jupyter_code>regions_folder = "/home/mlopez/EXEC/Grids-polygons-regions/"
files_regions = glob.glob(regions_folder+"*"+"lat_lon.feather")
files_regions
listdfreg=[]
for region in files_regions:
    df = pd.read_feather(region)
    listdfreg.append(df)<jupyter_output><empty_output><jupyter_text>Create DF by region<jupyter_code>short_dict = {"DDE_STF_20K_REG_FOR_VUE_S": ("NM_REG_FOR", "RF"), 
            "DDE_STF_20K_UA_PER_VUE_S": ("PER_NO_UA", "UA"), 
              "DOM_BIO": ("NOM", "DB"), 
              "REG_ECO": ("NOM", "RE"), 
              "SDOM_BIO": ("NOM", "SDB"), 
              "Secteurs_Operations_Regionales": ("D_GENERAL", "SOR"), 
              "SREG_ECO": ("NOM", "SRE"),  
              "territoire_guide": ("TER_GUIDE", "TG") 
              }

for region, (name, short) in short_dict.items():
    print(region, name, short)
def replace_all(text, dic):
    for i, j in dic.items():
        text = text.replace(i, j)
    return text
d = { 'BAS-SAINT-LAURENT':"Bas-Saint-Laurent", 
     'SAGUENAY -LAC-SAINT-JEAN': "Saguenay -Lac-Saint-Jean",
     'CAPITALE-NATIONALE-CHAUDIÈRE-APPALACHES':"Capitale-Nationale-Chaudiere-Appalaches",
     'MAURICIE-CENTRE-DU-QUÉBEC':'Mauricie-Centre-du-Quebec','OUTAOUAIS':'Outaouais', 
     'ABITIBI-TEMISCAMINGUE':'Abitibi-Temiscamingue', 'COTE-NORD':'Cote-Nord',
     'NORD-DU-QUEBEC':'Nord-Du-Quebec', 'GASPESIE-ILES-DE-LA-MADELEINE':'Gaspesie-Iles-De-La-Madeleine', 
     'LANAUDIERE':'Lanaudiere','LAURENTIDES':"Laurentides", 
     "ESTRIE-MONTÉRÉGIE-LAVAL-MONTRÉAL":"Estrie-Monteregie-Laval-Montreal", 
     "é": "e", "É": "E", "à": "a", "è": "e", "Î": "i", "È": "E", "ô" : "o", "Ç":"C", "ç":"c",
       }<jupyter_output><empty_output><jupyter_text>Cutting rcp45 and 85 by region and getting df with 4th degree polynomial<jupyter_code>def polysmooth(years, y, degree):
    coef = np.polyfit(years, y, degree)
    return np.polyval(coef,years)
seasonlist = ["winter", "spring", "summer", "fall"]
for region, (name, short) in short_dict.items():
    print("------------------------------------------------------------------")
    print ("1/8... Cutting by region")
    print(region, name)
    dfregion = pd.read_feather("/home/mlopez/EXEC/Grids-polygons-regions/"+region+"lat_lon.feather")
    print ("2/8... Merging region with data")
    dfdatareg45 = pd.merge(dfregion, df2m, on=["lat","lon"])
    dfdataregObs = pd.merge(dfregion, dfObs, on=["lat","lon"])
    for seas in seasonlist:
        print (seas)
        print ("3/8... Get mean by year for every sub-region for every season")
        dfmeanUA45 = dfdatareg45.loc[dfdatareg45['season'] == seas]
        dfmean45UA = dfmeanUA45.groupby(["time", name, 'season']).mean()
        dfmean45UA2 = dfmean45UA.reset_index()
        dfmean45UA=0
        dfmeanObs = dfdataregObs.loc[dfdataregObs['season'] == seas]
        dfmeanObs2 = dfmeanObs.groupby(["time", name, 'season']).mean() 
        dfmeanObs =0
        dfmeanObs3 = dfmeanObs2.reset_index()
        dfmeanObs2=0
        print("Merging 3 cases")
        df3m = pd.merge(dfmean45UA2, dfmeanObs3, how='left', on= ["time", name, "season"] )
        dfmeanObs3=0
        dfmean45UA2=0
        print ("4/8... Split by subregion")
        df3ma = df3m[['time', name, var+'_p10_45', var+'_p50_45', var+'_p90_45',
                                   var+'_p10_85', var+'_p50_85', var+'_p90_85', var+'_Obs', "season"]]
        df3m=0
        if (var == "tg_mean" or var== "tn_mean" or var=="tx_mean"):
            df3ma[var+"_Obs"] = df3ma[var+"_Obs"]- 273.15
        if (var == "solidprcptot"):
            df3ma[var+"_Obs"] = df3ma[var+"_Obs"]/10
        df3ma[var+"_Obs"] = round(df3ma[var+"_Obs"], 2)
        for name2,dfsr in df3ma.groupby(name):
            print ("                   "+name2, name)
            dfsrc = dfsr.copy()
            print ("6/8... 4th Polynomial - rcp45,85")
            years = dfsrc.time.dt.year
            dfsrc["time"] = years#.values
            for p in ('_p10_45', '_p50_45', '_p90_45', '_p10_85', '_p50_85', '_p90_85'):
                if (var == "tg_mean" or var== "tn_mean" or var=="tx_mean"):
                    dfsrc[var+p] = polysmooth(years,dfsrc[var+p],4) - 273.15
                if (var == "solidprcptot"):
                    dfsrc[var+p] = polysmooth(years,dfsrc[var+p],4)/10
                if (var == "prcptot", "dlyfrzthw", "DJC"):
                    dfsrc[var+p] = polysmooth(years,dfsrc[var+p],4)
                dfsrc[var+p] = round(dfsrc[var+p], 2) 
            print(" -----------------------8/8 Creating CSV file for sub-region " +name2)
            dfsrc.to_csv(replace_all(name2,d)+"_"+var+"_"+seas+".csv")

dfdatareg45 = 0
dfregion=0
df2m=0
dfdataregObs=0
dfObs=0<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17546090273363002, 'fraction_numerical': 0.05096418732782369, 'mean_word_length': 3.5184298707515556, 'pattern_counts': {'":': 17, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '324311', 'license': 'Apache-2.0', 'n_tokens_mistral': 4092, 'n_tokens_neox': 3755, 'n_words': 647, 'path': 'Data Processing Jupyter Notebooks/DF for Figure in R 7 variables  seasonal  all regions.ipynb', 'repo_name': 'Ouranosinc/PortraitsClimMFFP', 'size': '9438'}