<jupyter_start><jupyter_text>Work with Pyemu to make .pst object and stuff....  Using MAP SWB observations and parameters<jupyter_code># First run-through 11/26/2019
# More parameters 1/29/2020
# More observations 2/3/2020
# Lots of updates July 2020:
#   - new lookup table
#   - adjustments to observations sorting and numbers

# 8/10/2020 Added lots more observations - SW watersheds baseflow and runoff.
# 8/20/20 Re-worked to create more generalized process, moved weights to separate notebook

# 9/9/2020 Ran with full set of MAP parameters (494 instead of 56)
# Reinstalled pyemu 7/15/2020 with pip
import pyemu as pyemu
import os
import pandas as pd
pyemu.__version__<jupyter_output><empty_output><jupyter_text>Using the "from_io" utility<jupyter_code># Set up paths:
PARDIR = 'E:\\UMID_Data\\0188_offline\\modeldev\\PARS'
OBSDIR = 'E:\\UMID_Data\\0188_offline\\modeldev\\OBS'
outdir = 'E:\\UMID_Data\\0188_offline\\modeldev\\PEST\\PEST_FILE_UPDATES'<jupyter_output><empty_output><jupyter_text>IMPORTANT TO UPDATE THESE FILE NAMES:<jupyter_code># Set current versions of lookup tables:
#landuse_version = 'YETI_Landuse_lookup_table_V4'
#irrigation_version = 'YETI_Irrigation_lookup_table_V2'
landuse_version = 'YETI_LU_lookup_table_MOREPARS'
irrigation_version = 'YETI_IRR_lookup_table_MOREPARS'

lufile = landuse_version + '.xlsx'
irrfile = irrigation_version + '.xlsx'<jupyter_output><empty_output><jupyter_text>Run notebook 1 again if parameters/values have changed<jupyter_code># Lookup tables saved from jupyter notebook 1.Export lookup tables and tpl files from excel.ipynb:

# LAND USE infiles
infile = os.path.join(outdir, (landuse_version + '.txt')) 
tplfile = os.path.join(outdir,(landuse_version + '.tpl'))

# Irrigation ET in-file NEW_Irrigation_lookup_table_PestTest.txt:
infile2 = os.path.join(outdir,(irrigation_version + '.txt'))
tplfile2 = os.path.join(outdir,(irrigation_version + '.tpl'))

# Observations - TWO types now:
# in the P:\0188\modeldev\OBS directory:
# "ins_file" = MAP_simulated_et_obs.ins
insfile1 = os.path.join(OBSDIR,'MAP_simulated_et_obs_ALL_ET_sorted.ins')
insfile2 = os.path.join(OBSDIR,'MAP_simulated_obs_SW.ins')

# "out_file" = swbstats_simulated_obs_et.obs
obsfile1 = os.path.join(OBSDIR,'swbstats_simulated_obs_et_ALL_sorted.obs')
obsfile2 = os.path.join(OBSDIR,'swbstats_simulated_obs_SW.obs')

# OTHER data:
# obsdir --> \ALLOBS.INFO.for.pyemu.csv  --> OBSVAL, OBSNME, WEIGHT, has lots of
# entries that are "zero" values and will be skipped

# pardir -->\MAP_parlist.csv --> PARNME, PARVAL1, PARGPNME, PARLBND, PARUBND
#mypst = pyemu.pst.pst_utils.pst_from_io_files(
#    tpl_files=[tplfile], in_files=[infile], ins_files=[insfile], out_files=[obsfile])
mypst = pyemu.helpers.pst_from_io_files(
    tpl_files=[tplfile,tplfile2], in_files=[infile,infile2], ins_files=[insfile1,insfile2], out_files=[obsfile1,obsfile2])
print (mypst.observation_data.head())
print (mypst.observation_data.info())
print (mypst.parameter_data.tail())
print (mypst.parameter_data.info())
# For some reason this is not the same number of pars as in the par info notebook.<jupyter_output>parnme partrans parchglim  parval1       parlbnd  \
wetf_d_tew      wetf_d_tew      log    factor      1.0  1.100000e-10   
wetf_height    wetf_height      log    factor      1.0  1.100000e-10   
wetf_kcb-mid  wetf_kcb-mid      log    factor      1.0  1.100000e-10   
wht_kcb-mid    wht_kcb-mid      log    factor      1.0  1.100000e-10   
wtem_kcb-mid  wtem_kcb-mid      log    factor      1.0  1.100000e-10   

                   parubnd  pargp  scale  offset  dercom  
wetf_d_tew    1.100000e+10  pargp    1.0     0.0       1  
wetf_height   1.100000e+10  pargp    1.0     0.0       1  
wetf_kcb-mid  1.100000e+10  pargp    1.0     0.0       1  
wht_kcb-mid   1.100000e+10  pargp    1.0     0.0       1  
wtem_kcb-mid  1.100000e+10  pargp    1.0     0.0       1  
<class 'pandas.core.frame.DataFrame'>
Index: 494 entries, bare_a_cn to wtem_kcb-mid
Data columns (total 10 columns):
 #   Column     Non-Null Count  Dtype  
---  ------     --------------  -----  
 0   parnme     [...]<jupyter_text>Add my own OBS and PAR data to the PST object: Adding in SW obs info**Will need another section for bounds on the Irrigation multipliers...**<jupyter_code># Get my own OBS data....
# Uses output from notebook 4.Set and Update OBS weights

allobs = pd.read_csv(os.path.join(OBSDIR,'ALLOBS.INFO.for.pyemu.csv'))

del allobs['Unnamed: 0']
print (allobs.info())
print (allobs.tail(10))
# Join my OBS data with the .pst obs data:
# This makes sure everything is sorted correctly...
# fewer values than the .csv file because a lot that were generated are zeros
newpd = pd.merge(mypst.observation_data, allobs, how='left', left_on='obsnme', right_on='ObsName')
print (newpd.info())
# The unmatched values are SW vals where there wasn't any ACTUAL gage data.
# Need to ZERO weight these, since there's nothing to compare them to
## Unweight non-matched values:
newpd.loc[newpd.ObsName.isnull(), 'WEIGHT'] = 0
newpd.loc[newpd.ObsName.isnull(), 'OBS_group'] = 'NO_SW_Data'
newpd.loc[newpd.ObsName.isnull(), 'Value'] = -999

#newpd.loc[newpd.ObsName.isnull()].to_csv('MISSING_SW_OBS.csv')
# with the sort correct, this works:
mypst.observation_data['obsval'] = newpd.loc[:,'Value'].values
mypst.observation_data['weight'] = newpd.loc[:,'WEIGHT'].values
mypst.observation_data['obgnme'] = newpd.loc[:,'OBS_group'].values   # Can assign other groups to test as needed....

print (mypst.observation_data.head(10))<jupyter_output>obsnme  obsval    weight    obgnme
aetc13-301_2013_04  aetc13-301_2013_04    4.02  4.285403  aet-corn
aetc13-301_2013_05  aetc13-301_2013_05    4.55  3.377105  aet-corn
aetc13-301_2013_06  aetc13-301_2013_06    4.99  3.143509  aet-corn
aetc13-301_2013_07  aetc13-301_2013_07    6.04  6.887002  aet-corn
aetc13-301_2013_08  aetc13-301_2013_08    5.82  3.371081  aet-corn
aetc13-301_2013_09  aetc13-301_2013_09    1.99  6.801503  aet-corn
aetc13-302_2013_04  aetc13-302_2013_04    3.58  4.285403  aet-corn
aetc13-302_2013_05  aetc13-302_2013_05    4.22  3.377105  aet-corn
aetc13-302_2013_06  aetc13-302_2013_06    5.87  3.143509  aet-corn
aetc13-302_2013_07  aetc13-302_2013_07    6.13  6.887002  aet-corn<jupyter_text>PARAMETERS.... from LU and IRR tables<jupyter_code># Get my own lookup table PAR data....
# Made this new 8/10/2020

pardata1 = pd.read_csv(os.path.join(PARDIR,'MAP_parlist_MOREPARS.csv')).drop_duplicates(['ParName','ParGroup'])
# Recast to lower case:
pardata1['ParName'] = pardata1['ParName'].str.lower()
print (pardata1.info())
print (pardata1.head())
print (mypst.parameter_data.info())
# Join these together:
pardf = pd.merge(mypst.parameter_data, pardata1, how='left', left_on='parnme', right_on='ParName')
#print (pardf.info())

# Checking for mis-matched values between the merged datasets:
# THIS ALSO LISTS THE NEW PARAMETERS FROM THE IRRIGATION MULTIPLIERS....

print (pardf.loc[pardf.parnme.isnull()])
# Assign values below....
# JOIN the PARDATA into the pst object:
# Join to pardf....

# with the sort correct, this works:
mypst.parameter_data['parval1'] = pardf.loc[:,'ParVal'].values
mypst.parameter_data['parlbnd'] = pardf.loc[:,'ParLBND'].values
mypst.parameter_data['parubnd'] = pardf.loc[:,'ParUBND'].values
mypst.parameter_data['pargp'] = pardf.loc[:,'ParGroup'].values   


print(mypst.parameter_data.head())
print (mypst.parameter_data.loc[mypst.parameter_data.parval1.isnull()])
# Fix these later if there are any...<jupyter_output>Empty DataFrame
Columns: [parnme, partrans, parchglim, parval1, parlbnd, parubnd, pargp, scale, offset, dercom]
Index: []<jupyter_text>Setting parameter transformation info (setting non-adjustable parameters, etc.)<jupyter_code># mess with pardf first, then copy into mypst.parameter_data
# Have everything that can vary be LOG transformed (this is what is recommended)
pardf['PARTRANS'] = 'x'
pardf.loc[pardf.Adjust == 'n', 'PARTRANS'] = 'fixed'
pardf.loc[pardf.Adjust == 'YES', 'PARTRANS'] = 'log'
mypst.parameter_data['partrans'] = pardf.loc[:,'PARTRANS'].values
# How many are adjustable?
print (pardf.loc[pardf['Adjust']=='YES'].info())<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 494 entries, 0 to 493
Data columns (total 18 columns):
 #   Column      Non-Null Count  Dtype  
---  ------      --------------  -----  
 0   parnme      494 non-null    object 
 1   partrans    494 non-null    object 
 2   parchglim   494 non-null    object 
 3   parval1     494 non-null    float64
 4   parlbnd     494 non-null    float64
 5   parubnd     494 non-null    float64
 6   pargp       494 non-null    object 
 7   scale       494 non-null    float64
 8   offset      494 non-null    float64
 9   dercom      494 non-null    int32  
 10  Unnamed: 0  494 non-null    int64  
 11  ParName     494 non-null    object 
 12  ParGroup    494 non-null    object 
 13  ParVal      494 non-null    float64
 14  ParUBND     494 non-null    float64
 15  ParLBND     494 non-null    float64
 16  Adjust      494 non-null    object 
 17  PARTRANS    494 non-null    object 
dtypes: float64(8), int32(1), int64(1), object(8)
memory usage: 71.4+ KB
No[...]<jupyter_text>Temporary testing of changing parameter values....<jupyter_code>mypst.parameter_data.pargp.unique()<jupyter_output><empty_output><jupyter_text>Some cleanup after running PESTCHEK:  More cleanup after pestchek.exe  Checking for pars set at or beyond bounds, fix if needed:<jupyter_code>## CHECK FOR PARAMETERS ABOVE UPPER BOUND....  If not zero, need to fix.

mypst.parameter_data.loc[((mypst.parameter_data.parval1 > mypst.parameter_data.parubnd))] 
## CHECK FOR PARAMETERS **AT** UPPER BOUND.... (and adjustable)
mypst.parameter_data.loc[((mypst.parameter_data.parval1 == mypst.parameter_data.parubnd) & (mypst.parameter_data.partrans == 'log'))] 
## CHECK for parameters equal to their lower bounds and adjustable...
# This will cause problems for the regularization.
# If not zero, need to fix:

mypst.parameter_data.loc[((mypst.parameter_data.parval1 == mypst.parameter_data.parlbnd) & (mypst.parameter_data.partrans == 'log'))] 

# Need to check for nans in dataframe.... If there are any, need to fix:
mypst.parameter_data.isnull().any()
mypst.parameter_data.loc[mypst.parameter_data.parval1.isnull()]<jupyter_output><empty_output><jupyter_text>Copy FINAL versions of par files to PEST directory:<jupyter_code># These will be copied over to YETI for running
# E:\UMID_Data\0188_offline\modeldev\PEST\PEST_FILE_UPDATES
import shutil
insfiles = [f for f in os.listdir(OBSDIR) if f.endswith('.ins')]
print (insfiles)
for f in insfiles:
    shutil.copy(os.path.join(OBSDIR, f), outdir)<jupyter_output>['MAP_simulated_et_obs.ins', 'MAP_simulated_et_obs_ALL_ET.ins', 'MAP_simulated_et_obs_ALL_ET_sorted.ins', 'MAP_simulated_obs_SW.ins']<jupyter_text>Final setting of commands and files, NOPTMAX:<jupyter_code>mypst.control_data.noptmax = -1
mypst.control_data.formatted_values
mypst.model_command = './swb_forward_run_ALL_obs.sh'      # This runs swb and everything else
mypst.control_data.noptmax = 0<jupyter_output><empty_output><jupyter_text>Make sure these are updated:<jupyter_code>#(landuse_version + '.txt')
#(landuse_version + '.tpl')
mypst.template_files = [landuse_version + '.tpl', irrigation_version + '.tpl']
mypst.input_files = [landuse_version + '.txt', irrigation_version + '.txt']

mypst.instruction_files = ['MAP_simulated_et_obs_ALL_ET_sorted.ins','MAP_simulated_obs_SW.ins']
mypst.output_files = ['swbstats_simulated_obs_et_ALL_sorted.obs','swbstats_simulated_obs_SW.obs']
mypst.model_command
#mypst.model_input/output<jupyter_output><empty_output><jupyter_text>Write output:<jupyter_code>mypst.write(os.path.join(outdir,'MAP_allobs_calib_run_allpars.pst'))
# Need to edit line 8 of this to match prior versions that worked - remove last two variables.
mypst.nobs<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11860132806589897, 'fraction_numerical': 0.0590905270236194, 'mean_word_length': 4.285650821856953, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '313978', 'license': 'CC0-1.0', 'n_tokens_mistral': 4938, 'n_tokens_neox': 4447, 'n_words': 1004, 'path': 'src/jupyter/5.Pyemu for MAP project - make PST files from input.ipynb', 'repo_name': 'smwesten-usgs/swb2_pestpp', 'size': '11897'}