<jupyter_start><jupyter_text>Am noting that the decline in water vapor intensity is not being preserved in predictions from successive calibration data.  **Want to know how to get better model fidelity. ***All this comes from http://austinrochford.com/posts/2015-08-29-additive.html<jupyter_code>from matplotlib import pyplot as plt
import numpy as np
import pandas as pd
import patsy
import scipy as sp
import seaborn as sns
from statsmodels import api as sm

from pygam import LinearGAM
from pygam.utils import generate_X_grid
from itertools import chain
pth = '/Users/phora/GDrive/research/ZIPP/ZIPP2_EN602/EN602_Loose/science/UMS/MS Data/'

massspec_12_1 = pd.read_table(pth+'MSData_7_12_2017 21_08.dfData',sep='\t',parse_dates=[0], header=0, low_memory=False)
massspec_12_1.dropna(axis=0, how='all')
ms = massspec_12_1.iloc[5858:9895, :]


hdrs = ms.columns.values
idx = [62,5]
ms = ms.dropna(how='any')
#print(hdrs[idx])
df = pd.DataFrame(ms[hdrs[idx]])
target_df = pd.Series(ms[hdrs[64]])

### Add scaling values
s18 = 1e-9
s32 = 1e-10

X = df
y = target_df

### Make a division between test and train data:
#from sklearn.model_selection import train_test_split
X_train = X
y_train = y

lam = [0.1]
#lam = [0.01,0.01,1,0.01,1,1]
#lam = [1,1,1,1,1,1]
#gam = LinearGAM(n_splines=5,lam=lam,penalties='derivative').fit(X_train, y_train)
gam = LinearGAM(n_splines=10).gridsearch(X, y)

gam.summary()
XX = generate_X_grid(gam)
plt.rcParams['figure.figsize'] = (18, 8)
fig, axs = plt.subplots(1, len(hdrs[idx]))
titles = hdrs[idx]
for i, ax in enumerate(axs):
    pdep, confi = gam.partial_dependence(XX, feature=i+1, width=.95)
    ax.plot(XX[:, i], pdep)
    #ax.plot(XX[:, i], confi[0][:, 0], c='grey', ls='--')
    #ax.plot(XX[:, i], confi[0][:, 1], c='grey', ls='--')
    #ax.set_title(titles[i],fontsize=12)
plt.show()
pdep
f1,ax = plt.subplots();

plt.plot(y.as_matrix())
yy = gam.predict(X) 
plt.plot(yy,'r')
#plt.ylim(7e-11,9e-11)
#plt.ylim(0.02,0.06)
plt.ylabel('Ion Current (mA)')
ax2 = ax.twinx()

err = np.sqrt((y.as_matrix()-yy)**2)/y.as_matrix()*100
plt.plot(err,'g')
plt.ylim(0,5)
plt.ylabel('Error in %')
plt.show()
#Now try to test on the next day's calibration data
#fil = 'MSData_7_14_2017 21_13.dfData'
massspec_15_1 = pd.read_table(pth+'MSData_7_15_2017 22_44.dfData',sep='\t',parse_dates=[0], header=0, low_memory=False,encoding='latin1')
mss = massspec_15_1.iloc[4064:6883, :]
mss = mss.dropna(how='any')

hdr15 = mss.columns.values

X15 = pd.DataFrame(mss[hdr15[idx]])
Y15 = pd.Series(mss[hdr15[64]])

X15 = X15/s18
Y15 = Y15/s32
## Check to see if water vapor dropped as much in the subsequent cal.
f2,ax2 = plt.subplots(figsize=(25,10));


#plt.plot(mss[hdrs[0]],mss[hdrs[idx[0]]],'r')
#plt.plot(ms[hdrs[0]],ms[hdrs[idx[0]]],'g')
#plt.ylim(2e-11,13e-11)
#plt.ylim(0.01,0.1)
#plt.ylabel('Ion Current (mA)')
#ax3 = ax2.twinx()

yy15 = gam.predict(X15) 
plt.plot(mss[hdrs[0]],yy15,'k')

plt.plot(mss[hdrs[0]],Y15.as_matrix(),'b')
plt.plot(ms[hdrs[0]],ms[hdrs[64]]/s32,'y')
#err = np.sqrt((Y15.as_matrix()-yy15)**2)/Y15.as_matrix()*100
#plt.plot(err,'g')
#plt.ylim(0,50)
#plt.ylabel('Error in %')
plt.show()

d18 = (2.35-2.0)/2.35

dact = (9.5-8.25)/9.5

dpred = (9.5-9.0)/9.5

d18
f2,ax2 = plt.subplots();

plt.plot(Y15.as_matrix())

yy15 = gam.predict(X15) 
plt.plot(yy15,'r')
#plt.ylim(2e-11,13e-11)
#plt.ylim(0.01,0.1)
plt.ylabel('Ion Current (mA)')
ax3 = ax2.twinx()

err = np.sqrt((Y15.as_matrix()-yy15)**2)/Y15.as_matrix()*100
plt.plot(err,'g')
plt.ylim(0,50)
plt.ylabel('Error in %')
plt.show()
from scipy import signal as sig
from scipy.interpolate import griddata
import gsw
#NOTES FROM SAM
#back=np.average(mass32_cal)/4.91793e-11
#corr32 = np.average(mass32_cal/cal_avg)
#conc_2=208.51120836118787
#CF = conc_2 / (corr32-back)
#cal32 = ((mass32/fitted32)-back)*CF







import patsy as pt
from rpy2.robjects import r, pandas2ri
pandas2ri.activate()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19161982626469085, 'fraction_numerical': 0.08610117526826776, 'mean_word_length': 5.418032786885246, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '600494', 'license': 'MIT', 'n_tokens_mistral': 1979, 'n_tokens_neox': 1785, 'n_words': 325, 'path': 'notebooks/pyGAM_fit_to_vapor_only.ipynb', 'repo_name': 'bloose/bias_correction_by_ML', 'size': '3914'}