<jupyter_start><jupyter_text>Apply the global irrigation topology to get mean monthly demandsInne Vanderkelen - March 2021<jupyter_code># import modules
import numpy as np
import geopandas as gpd
import pickle
import xarray as xr
import warnings
import pandas as pd

warnings.filterwarnings('ignore')

# add path where utils modules are located to python path
import sys
sys.path.append('C:\\Users\\ivand\\OneDrive - Vrije Universiteit Brussel\\PhD\\3_reservoir_release\\')

# import own functions
import pfaf.pfafstetter as pfaf # decode package from Naoki, see https://github.com/nmizukami/pfaf_decode 
from utils_plotting import *
from utils_irrigtopo import *


# Settings: 
outlet_threshold = 700e3 # in m
tributary_threshold = 100e3 # m

data_dir = 'C:\\Users\\ivand\\OneDrive - Vrije Universiteit Brussel\\PhD\\3_reservoir_release\\data_for_mizuroute\\'

# define reservoir dependency dataset
res_dependency = 'res_dependency_HDMA_outletthres_700km_tribthres_100km.pkl'
# calculate or load reservoir dependency 
calc_res_dependency = False # if True: Calculate, if False: load

# irr demand sensitivity
sensitivity="_obsscaled"<jupyter_output><empty_output><jupyter_text>1. Open river topology and get hrus with ids<jupyter_code># open river topology and get hrus with ids
ntopo = xr.open_dataset(data_dir+'ntopo_hdma_mod.reorder_lake_H06.nc')

# get pfafs corresponding to segments to hrus trough hru_seg_id
seg_pfafs = ntopo[['PFAF','seg_id']].to_dataframe()
# correct for spaces in front of the PFAF code
seg_pfafs['PFAF'] = np.char.strip( seg_pfafs['PFAF'].values.astype(str))

hrus_ids = ntopo[['hru_seg_id','hruid','Basin_Area']].to_dataframe()
hrus = hrus_ids.merge(seg_pfafs,left_on='hru_seg_id',right_on=('seg_id'))<jupyter_output><empty_output><jupyter_text>2. Load irrigation amounts per hru and merge to hru dataframe<jupyter_code># load remapped irrigation as pandas dataframe and add basinIrrig to main hrus dataframe
ds_hru_irrig = xr.open_dataset(data_dir + 'CLM_HDMA_remapped_monmean_1971-2000'+sensitivity+'.nc')
# rename variable per month
months = ['Jan','Feb','Mar','Apr','May','Jun','Jul','Aug','Sep','Oct','Nov','Dec']
ds_hru_irrig_month = xr.Dataset()
for i,month in enumerate(months): 
    da_month = ds_hru_irrig['QIRRIG'][i,:]
    ds_hru_irrig_month['QIRRIG_'+month] = da_month

hru_irrig = ds_hru_irrig_month.to_dataframe()
hru_irrig = hru_irrig.merge(pd.read_csv(data_dir+'remap/lookup_ID_hruid.csv').drop_duplicates(), on='ID')
hrus_irrig = hrus.merge(hru_irrig, on='hruid')

# convert from mm/s to m³/s by multiplying with basin area
for month in months: 
    hrus_irrig['QIRRIG_'+month] =  hrus_irrig['QIRRIG_'+month] * 10**(-3) * hrus_irrig['Basin_Area']
# open res dependency
with open(data_dir+"irrigation_topology/"+res_dependency, 'rb') as handle:
    res_dependency_dict = pickle.load(handle)
  
%%time

demand_df = pd.DataFrame(index=list(res_dependency_dict.keys()))
demand_df.index.name = 'PFAF'

for month in months: 

    print('', end='\n' )
    print(month, end='\n' )

    demand_month = [] 
    count = 1

    for reservoir in res_dependency_dict: 
        print('Calculating demand for reservoir '+str(count)+ ' of '+str(len(res_dependency_dict)),end='\r' )

        demand_tot = 0
        # calculate sum over all river segments with weights applied
        seg_to_sum = list(res_dependency_dict[reservoir].keys())
        weights = list(res_dependency_dict[reservoir].values())
        demand_tot = (hrus_irrig.loc[hrus_irrig['PFAF'].isin(seg_to_sum), 'QIRRIG_'+month].values * weights).sum()
        # store in dict per reservoir
        demand_month.append(demand_tot)

        count = count+1
    demand_df['H06_D_'+month] = demand_month

# replace inf values with original demand values. 
demand_df_or = pd.read_csv(data_dir+'reservoirs_monthly_demand.csv', index_col='PFAF')    
demand_df_nan = demand_df.replace(np.inf, np.nan)
demand_df_nan.index = demand_df_nan.index.astype('int64')
demand_df_nan = demand_df_nan.combine_first(demand_df_or)


demand_df_nan.to_csv(data_dir+'reservoirs_monthly_demand'+sensitivity+'.csv')<jupyter_output>Jan
Calculating demand for reservoir 484 of 484
Feb
Calculating demand for reservoir 484 of 484
Mar
Calculating demand for reservoir 484 of 484
Apr
Calculating demand for reservoir 484 of 484
May
Calculating demand for reservoir 484 of 484
Jun
Calculating demand for reservoir 484 of 484
Jul
Calculating demand for reservoir 484 of 484
Aug
Calculating demand for reservoir 484 of 484
Sep
Calculating demand for reservoir 484 of 484
Oct
Calculating demand for reservoir 484 of 484
Nov
Calculating demand for reservoir 484 of 484
Dec
Wall time: 3min 14s
{'fraction_non_alphanumeric': 0.10176991150442478, 'fraction_numerical': 0.02338811630847029, 'mean_word_length': 4.9411764705882355, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '144590', 'license': 'MIT', 'n_tokens_mistral': 1861, 'n_tokens_neox': 1653, 'n_words': 396, 'path': 'preprocessing/apply_irrigtopo_monthlydemand.ipynb', 'repo_name': 'VUB-HYDR/2022_Vanderkelen_etal_GMD', 'size': '4746'}