<jupyter_start><jupyter_text>Baseflow from: https://engineering.purdue.edu/mapserve/WHAT/  Monthly Data<jupyter_code>rootpath = 'C:/Users/PAULINKENBRANDT/Downloads/'

runoff_file = rootpath + "Wed_Oct_25_12_00_59_2017__8896_monthly-directrunoff.csv"
baseflow_file = rootpath + "Wed_Oct_25_12_00_59_2017__8896_monthly-baseflow.csv"
flow_file = rootpath + "Wed_Oct_25_12_00_59_2017__8896_monthly-flow.csv"
def fix_dates(flowfile,flowtype):
    runoff = pd.read_csv(flowfile, skiprows=5,
                         names=['yr','10oct','11nov','12dec','01jan','02feb','03mar','04apr','05may',
                                '06jun','07jul','08aug','09sep','total'],
                         usecols=[0,1,2,3,4,5,6,7,8,9,10,11,12])

    runoff['year'] = runoff['yr'].apply(lambda x: x[-4:])
    runoff.loc[:,['10oct','11nov','12dec']] = runoff[['10oct','11nov','12dec']].shift(1)
    runoff = runoff.drop(['yr'],axis=1).sort_index(axis=1).set_index('year')
    runoff = runoff.unstack().reset_index()
    runoff['month'] = runoff['level_0'].apply(lambda x: x[0:2],1)

    runoff['yrmo'] = runoff[['year','month']].apply(lambda x: pd.datetime(int(x[0]),int(x[1]),1),1)
    runoff.drop(['level_0','year','month'],axis=1,inplace=True)
    runoff.set_index('yrmo',inplace=True)
    runoff.columns = [flowtype]
    runoff[flowtype] = runoff[flowtype].apply(lambda x: round(x * 0.0283168,2),1)
    return runoff
runoff = fix_dates(runoff_file,'runoff')
baseflow = fix_dates(baseflow_file,'baseflow')
flow = fix_dates(flow_file,'total flow')

flow_data = pd.concat([runoff,baseflow,flow],axis=1)
flow_data['perc_base'] = flow_data['baseflow']/flow_data['total flow']
flow_data = flow_data[(flow_data.index >= pd.datetime(2003,1,1))&(flow_data.index <= pd.datetime(2016,12,1))]

flow_data.sort_index(inplace=True)
flow_data['sequence'] = range(1,len(flow_data)+1)
flow_data = flow_data.reset_index().set_index('sequence')
flow_data['rowlab'] = flow_data['yrmo'].apply(lambda x: 'FLOW_OUT_{:%m}_{:%Y}'.format(x,x),1)
flow_data[['rowlab','total flow','baseflow']].to_clipboard(sep='\t',)
flow_data[['rowlab','total flow']].to_clipboard(sep='\t')
flow_data[['rowlab','total flow','baseflow']]
rchtxt = r"H:\GIS\Ogden_Valley\SWAT_2017_09_11\Scenarios\Complex_HRU_09_13_2017\rch.txt"
#rch = pd.read_csv("H:/GIS/Ogden_Valley/SWAT_09_2017/rch.txt")
rch = pd.read_csv(rchtxt)

rch['FLOW_IN_acftmo'] = 2130.56*rch["FLOW_INcms"]
rch['FLOW_OUT_acftmo'] = 2130.56*rch["FLOW_OUTcms"]

mean_monthly_data = rch.groupby(['SUB','MON']).median().reset_index()
yearly_data = mean_monthly_data.groupby(['SUB']).sum().reset_index()

yearly_data[yearly_data['SUB']==55][['FLOW_IN_acftmo','FLOW_OUT_acftmo']]

rch2016 = rch[(rch['YEAR']==2009)&(rch['SUB']==55)]
rch2016['FLOW_OUT_acftmo'].sum()
rch2016 = rch[rch['YEAR']==2016]
rch_Swat_CUP = r'H:\GIS\Ogden_Valley\SWAT8\ResRight.Sufi2.SwatCup\output.rch'
rch_Swat_CUP = r"H:/GIS/Ogden_Valley/SWAT_2017_09_11/Scenarios/Default/TxtInOut/output.rch"

colms =["NM","SUB","GIS","MON","AREAkm2","FLOW_INcms","FLOW_OUTcms","EVAPcms","TLOSScms"]
rch_cup = pd.read_table(rch_Swat_CUP,sep='\s+',skiprows=9,usecols=[0,1,2,3,4,5,6,7,8],names=colms)
rch_cup['FLOW_IN_acftmo'] = 2130.56*rch_cup["FLOW_INcms"]
rch_cup['FLOW_OUT_acftmo'] = 2130.56*rch_cup["FLOW_OUTcms"]
mean_monthly_data = rch_cup.groupby(['SUB','MON']).median().reset_index()
yearly_data = mean_monthly_data.groupby(['SUB']).sum().reset_index()

rch_cup2016 = rch_cup[(rch_cup['SUB']==45)]
rcup = rch_cup2016[rch_cup2016['MON']<14]
rcup_mo = rcup.groupby(['MON']).median()

rch_cup['dates'] = ''
for sub in rch_cup['SUB'].unique():
    dates = pd.date_range('1/1/2003', periods=172, freq='MS')
    rch_cup[rch_cup['SUB']==sub]['dates'] = dates
    
#rcup['dates'] = dates
#rcup.set_index('dates',inplace=True)
#rcup.groupby(rcup.index.year).sum()
rch_cup['dates']
colms =["NM","SUB","GIS","MON","AREAkm2","FLOW_INcms","FLOW_OUTcms","EVAPcms","TLOSScms"]
rch_cup = pd.read_table(rch_Swat_CUP,sep='\s+',skiprows=9,usecols=[0,1,2,3,4,5,6,7,8],names=colms)
rch_cup['FLOW_IN_acftmo'] = 2130.56*rch_cup["FLOW_INcms"]
rch_cup['FLOW_OUT_acftmo'] = 2130.56*rch_cup["FLOW_OUTcms"]
mean_monthly_data = rch_cup.groupby(['SUB','MON']).median().reset_index()
yearly_data = mean_monthly_data.groupby(['SUB']).sum().reset_index()

rch_cup2016 = rch_cup[(rch_cup['SUB']==45)]
rcup = rch_cup2016[rch_cup2016['MON']>14].groupby(['MON']).mean()
rcup.FLOW_OUT_acftmo.mean()<jupyter_output><empty_output><jupyter_text>Daily RCH<jupyter_code>dailydata_file = "C:/Users/PAULINKENBRANDT/Downloads/Mon_Oct_30_13_26_49_2017__6735 (1).csv"

dailydata = pd.read_csv(dailydata_file, error_bad_lines=False)

dailydata.dropna(inplace=True)
dailydata.columns = [x.strip() for x in dailydata.columns]
dailydata['Date'] = pd.to_datetime(dailydata['Date'])
dailydata.set_index('Date',inplace=True)
dailydata = dailydata.resample('1D').mean()
dailydata = dailydata.interpolate(method='time')
dailydata = dailydata[dailydata.index < pd.datetime(2017,1,1)]
dailydata['Flow_cms'] = dailydata['Flow'].apply(lambda x: round(x*0.0283168,2),1)
dailydata['Base Flow_cms'] = dailydata['Base Flow'].apply(lambda x: round(x*0.0283168,2),1)
dailydata['sequence'] = range(1,len(dailydata)+1)
dailydata['rowlab'] = dailydata.index.map('FLOW_OUT_{0:%Y}_{0:%j}'.format)


swat_obs_dir = "H:/GIS/Ogden_Valley/SWAT_2017_09_11/swatcup/Daily20171030.Sufi2.SwatCup/SUFI2.IN/"
obs_rch_file = swat_obs_dir + 'observed_rch.txt'
obs_rch_sum_file = swat_obs_dir + 'var_file_rch.txt'
rch_sub_nos = [61]

header = """{:}     : number of observed variables\n""".format(len(sub_nos))

dataset_text = """\nFLOW_OUT_{:}   : this is the name of the variable and the subbasin number to be included in the objective function
{:}   : number of data points for this variable as it follows below. First column is a sequential number from beginning
      : of the simulation, second column is variable name and date (format arbitrary), third column is variable value.\n
"""

with open(obs_rch_file, 'w') as obs_rch:
    obs_rch.write(header)    
    for sub_no in sub_nos:
        dailysub = dailydata[['sequence','rowlab','Flow_cms']]        
        obs_rch.write(dataset_text.format(sub_no,len(dailysub)))
        obs_rch.write(dailysub.to_csv(path_or_buf=None, header=None, index=None, sep='\t'))
obs_rch.close()

with open(obs_rch_sum_file, 'w') as obs_rch_sum:
    rch_file_strs = "FLOW_OUT_{:}.txt\n"*len(sub_nos)
    obs_rch_sum.write(rch_file_strs.format(*sub_nos))
obs_rch_sum.close()<jupyter_output>b'Skipping line 2820: expected 5 fields, saw 6\nSkipping line 2821: expected 5 fields, saw 6\n'<jupyter_text>From USBR<jupyter_code>import urllib.request, json 
with urllib.request.urlopen("http://bit.ly/2ygBynS") as url:
    data = json.loads(url.read().decode())


parlist = ['Day.Sum.ReservoirEvaporation.af','Day.Inst.ReservoirElevation.feet',
           'Day.Avg.ReservoirRelease.cfs','Day.Avg.ReservoirInflow.cfs',
           'Day.Inst.ReservoirStorage.af']
reslist = ['CAUSEY RESERVOIR','PINEVIEW RESERVOIR']
resdata = {}

for res in reslist:
    for par in parlist:
        resdata[res+':'+par] = pd.DataFrame(data[res]['parameters'][par]['timeinterval']['Daily'])
        resdata[res+':'+par]['datetime'] = pd.to_datetime(resdata[res+':'+par]['datetime'])
        resdata[res+':'+par].set_index('datetime',inplace=True)
        resdata[res+':'+par].drop('flag',axis=1,inplace=True)
        
res_df = pd.concat(resdata,axis=1)
res_df.columns = res_df.columns.droplevel(-1)


# metric conversions
for col in res_df.columns:
    res_df[col] = pd.to_numeric(res_df[col])
    if '.cfs' in col:
        res_df[col[:-4]+".cms"] = res_df[col].apply(lambda x: round(x*0.0283168,2),1)
        res_df.drop(col,axis=1,inplace=True)
    elif '.af' in col:
        res_df[col[:-3]+".mcu"] = res_df[col].apply(lambda x: str(round(x*1233.48,0))[:-2],1)
        res_df.drop(col,axis=1,inplace=True)
    elif '.feet' in col:
        res_df.drop(col,axis=1,inplace=True)
res_df = res_df[res_df.index < pd.datetime(2017,1,1)]


res_df.sort_index(inplace=True)
res_df.resample('1D').mean()
res_df = res_df.interpolate(method='time')

res_df['sequence'] = range(1,len(res_df)+1)
res_df['rowlab'] = res_df.index.map('RES_VOL_{0:%Y}_{0:%j}'.format)
swat_obs_dir = "H:/GIS/Ogden_Valley/SWAT_2017_09_11/swatcup/Daily20171030.Sufi2.SwatCup/SUFI2.IN/"
obs_res_file = swat_obs_dir + 'observed_res.txt'
sub_nos = [43,56]
resnos = {43:'CAUSEY RESERVOIR',56:'PINEVIEW RESERVOIR'}
df_dict = {}

header = """{:}     : number of observed variables
""".format(len(sub_nos))

dataset_text = """
RES_VOL_{:}   : this is the name of the variable and the subbasin number to be included in the objective function
{:}   : number of data points for this variable as it follows below. First column is a sequential number from beginning
      : of the simulation, second column is variable name and date (format arbitrary), third column is variable value.

"""

with open(obs_res_file, 'w') as obs_res:
    obs_res.write(header)    
    for sub_no in sub_nos:
        df_dict[sub_no] = res_df[['sequence','rowlab',str(resnos.get(sub_no))+':Day.Inst.ReservoirStorage.mcu']]        
        obs_res.write(dataset_text.format(sub_no,len(df_dict[sub_no])))
        obs_res.write(df_dict[sub_no].to_csv(path_or_buf=None, header=None, index=None, sep='\t'))
obs_res.close()

with open(obs_res_sum_file, 'w') as obs_res_sum:
    res_file_strs = "RES_VOL_{:}.txt\n"*len(sub_nos)
    obs_res_sum.write(res_file_strs.format(*sub_nos))
obs_res_sum.close()

swat_obs_dir = "H:/GIS/Ogden_Valley/SWAT_2017_09_11/swatcup/Daily20171030.Sufi2.SwatCup/SUFI2.IN/"
observed_file = swat_obs_dir + 'observed.txt'

rch_sub_nos = [61]
sub_nos = [43,56]
resnos = {1:'CAUSEY RESERVOIR',2:'PINEVIEW RESERVOIR'}
df_dict = {}


observed_rch_head = """
3     : number of observed variables
5     : Objective function type, 1=mult,2=sum,3=r2,4=chi2,5=NS,6=br2,7=ssqr,8=PBIAS,9=KGE,10=RSR,11=MNS
0.5   : min value of objective function threshold for the behavioral solutions
1     : if objective function is 11=MNS (modified NS),indicate the power, p.

"""
observed_rch_txt = """
{:}_{:}   : this is the name of the variable and the subbasin number to be included in the objective function
1     : weight of the variable in the objective function
{:}     : Dynamic flow separation. Not considered if -1. If 1, then values should be added in the forth column below after observations
-1    : constant flow separation, threshold value. (not considered if -1)
1     : if separation of signal is considered, this is weight of the smaller values in the objective function
1     : if separation of signal is considered, this is weight of the larger values in the objective function
15    : percentage of measurement error
{:}    : number of data points for this variable as it follows below. First column is a sequential number from beginning
      : of the simulation, second column is variable name and date (format arbitrary), third column is variable value.\n
"""

with open(observed_file, 'w') as observed:
    observed.write(header)    
    for rsub_no in rch_sub_nos:
        observed.write(observed_rch_head)
        dailysub = dailydata[['sequence','rowlab','Flow_cms','Base Flow_cms']]        
        observed.write(observed_rch_txt.format('FLOW_OUT',rsub_no,1,len(dailysub)))
        observed.write(dailysub.to_csv(path_or_buf=None, header=None, index=None, sep='\t'))

    for sub_no in sub_nos:
        df_dict[sub_no] = res_df[['sequence','rowlab',str(resnos.get(sub_no))+':Day.Inst.ReservoirStorage.mcu']]        
        observed.write(observed_rch_txt.format('RES_VOL',sub_no,-1,len(df_dict[sub_no])))
        observed.write(df_dict[sub_no].to_csv(path_or_buf=None, header=None, index=None, sep='\t'))
    observed.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16961939476138002, 'fraction_numerical': 0.0425531914893617, 'mean_word_length': 5.737864077669903, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 14, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '470071', 'license': 'MIT', 'n_tokens_mistral': 4965, 'n_tokens_neox': 4647, 'n_words': 700, 'path': 'Flow Data Prep.ipynb', 'repo_name': 'inkenbrandt/Ogden-Valley', 'size': '11797'}