<jupyter_start><jupyter_text>Logistic Regression (Not working so far, NEED Help)<jupyter_code># fig, ax = plt.subplots(figsize=(5, 3))
# y = dat['fraction'].values
# x = dat['height'].values
# # Create the linear regression model
# LogRegr = linear_model.LogisticRegression() 
# # plotting the balance
# ax.scatter(dat['height'], dat['fraction'], label = 'balance default',color='r')

# # fit the linear model 
# LogRegr.fit(x.reshape(-1,1),y.reshape(-1,1))
# xres = np.array(x).reshape(-1,1)
# plt.plot(x,LogRegr.predict_proba(xres)[:,1],'o',label = 'fit')
# # Find the coefficients B0 and B1
# print('B0, B1: ',LogRegr.intercept_, LogRegr.coef_[0])
# ax.set_ylabel('Probability Default')
# ax.set_xlabel('Balance')
# ax.set_title('Probability Default vs. Balance')
# plt.legend()<jupyter_output><empty_output><jupyter_text>Ordinary Least squares Regression Model by StatsModels (Not working so far, NEED Help)<jupyter_code># # # Ordinary least squares regression model
# results = smf.ols(formula = 'auc ~ height*sigma', data = dat)
# # Fit the results
# resultfit = results.fit()
# # Inspect the results
# print(resultfit.summary())
inputs=['fraction','sigma','amplitude','fwhm','height','auc','labelencoded']
X_train=cal[inputs]
y_train = cal['center']
X_test=dat[inputs]
y_test= dat['center']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.05, random_state=1010)
X_train<jupyter_output><empty_output><jupyter_text>KNN<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
from sklearn.model_selection import train_test_split
KNNmodel = KNeighborsClassifier(n_neighbors=15) # initialize the model by choosing the number of neighbors (flexibility of model in this case)
# KNNmodel = KNeighborsClassifier(n_neighbors=1) minimum neighbord minimum predicted error
# remember the parabolic function of training vs. test set there is limit to the flexilibility vs. errors of the model
# Expected n_neighbors <= n_samples,  but n_samples = 15, n_neighbors = 20
# thus KNNmodel = KNeighborsClassifier(n_neighbors=15) is max
KNNmodel.fit(X_train, y_train) # feed the model some training data 
rate = KNNmodel.predict(X_train) == y_train # USE TO PREDICT NEW X DATA PUT INTO THE TRAINING MODEL
print('Training Error Rate:', np.mean(rate)) # COMPARING THE PREDICTED VALUES WITH ACTUAL RATE VALUES

rate = KNNmodel.predict(X_test) == y_test
print('Testing Error Rate:', np.mean(rate))
# rate.mean() ratio of trues 
rate.mean()
KNNmodel<jupyter_output><empty_output><jupyter_text>Knn visualiztion (not working, NEED help)<jupyter_code># # additional library we will use 
# from matplotlib.colors import ListedColormap

# # just for convenience and similarity with sklearn tutorial
# # I am going to assign our X and Y data to specific vectors
# # this is not strictly needed and you could use elements df for the whole thing!
# elements = dat
# X=elements[['height','auc']]
# print(X)
# #this is a trick to turn our strings (type of element / class) into unique 
# #numbers.  Play with this in a separate cell and make sure you know wth is 
# #going on!
# levels,labels=pd.factorize(elements.labels)
# y=levels
# print(levels)
# #This determines levelspacing for our color map and the colors themselves
# h=0.02
# cmap_light = ListedColormap(['#FFAAAA', '#AAFFAA', '#AAAAFF'])
# cmap_bold = ListedColormap(['#FF0000', '#00FF00', '#0000FF'])

# # in the sklearn tutorial two different weights are compared
# # the decision between "uniform" and "distance" determines the probability
# # weight.  "uniform" is the version presented in class, you can change to 
# # distance
# # weights='uniform'
# weights='distance'

# # I am actually refitting the KNN here. If you had a big data set you would
# # not do this, but I want you to have the convenience of changing K or 
# # weights here in this cell. Large training sets with many features can take 
# # awhile for KNN training! 

# K=5
# clf = KNeighborsClassifier(n_neighbors=K, weights=weights)
# clf.fit(X,y)

# # Straight from the tutorial - quickly read and see if you know what these 
# # things are going - if you are < 5 min until end then you should skip this part 

# # Plot the decision boundary. For that, we will assign a color to each
# # point in the mesh [x_min, x_max]x[y_min, y_max].
# x_min, x_max = elements.height.min() - 0.1  , elements.height.max() + 0.1
# y_min, y_max = elements.auc.min() - 0.1  , elements.auc.max() + 0.1  
# print(y_min, y_max, h)
# xx, yy = np.meshgrid(np.arange(x_min, x_max, h), 
#                      np.arange(y_min, y_max, h))

# Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])

# # Put the result into a color plot
# Z = Z.reshape(xx.shape)
# plt.figure(figsize=(4,4));
# #plt.pcolormesh(xx, yy, Z, cmap=cmap_light)
# plt.pcolormesh(xx, yy, Z, cmap=cmap_light,vmin=0,vmax=np.max(y))
# # Plot also the training points
# # This may be the 1st time you have seen how to color points by a 3rd vector
# # In this case y ( see c=y in below statement ). This is very useful! 
# plt.scatter(X.height, X.auc, c=y, cmap=cmap_bold)

# # Set limits and lebels 
# plt.xlim(xx.min(), xx.max())
# plt.ylim(yy.min(), yy.max())
# plt.xlabel('height')
# plt.ylabel('auc')<jupyter_output><empty_output><jupyter_text>Multiple Linear Regression<jupyter_code>train,test = train_test_split(dat, test_size=0.2, random_state=1000)
numdescriptors = 8
# train linear model of descriptors
yrange =  np.arange(1,numdescriptors,1)
for Y in yrange:
    MLR=linear_model.LinearRegression()
    MLR.fit(train[train.columns.values[0:Y]],train[train.columns.values[Y]])
    # WE are going to train using the first 8 values

    # make predictions on test and train set 
    trainpred=MLR.predict(train[train.columns.values[0:Y]])
    # predict the outputs using the training dataset
    testpred=MLR.predict(test[train.columns.values[0:Y]])
    # predict using test dataset
    #make parity plot 
    maxlimit  = train[train.columns.values[Y]].max()

    plt.figure(figsize=(7,7))
    plt.xlim([0,maxlimit]);
    plt.ylim([0,maxlimit]);
    print('This is for Y ='+str(Y))
    plt.scatter(train[train.columns.values[Y]],trainpred, label='Training')
    plt.scatter(test[train.columns.values[Y]],testpred,color='r', label='Test')
    plt.plot([0,maxlimit],[0,maxlimit],lw=4,color='black')
    plt.legend()
    plt.xlabel('Actual Output')
    plt.ylabel('Predicted Output')

    #calculate the test and train error
    
    print("Train error",mean_squared_error(train[train.columns.values[Y]],trainpred)) # MSE of training dataset
    print("Test error",mean_squared_error(test[train.columns.values[Y]],testpred))
    # usually the training error is less than the test error
plt.plot(x,yactual,color='red',lw='3',label='actual')
plt.plot(x,regr.predict(x.reshape(-1,1)),ls='--',label='fit')
plt.legend(loc='lower right')
fig, ax = plt.subplots(figsize=(5, 3))
ax.scatter(x, yactual, label='actual')
ax.scatter(x, yrand, label='measured')
ax.scatter(x, regr.predict(x.reshape(-1,1)), label='predicted')
fig, ax = plt.subplots(figsize=(5, 3))
ax.scatter(yrand, regr.predict(x.reshape(-1,1)), label='parity')
fig, ax = plt.subplots(figsize=(5, 3))
ax.scatter(x, regr.predict(x.reshape(-1,1)) - yrand)
ax.plot(x, 0*x)<jupyter_output><empty_output><jupyter_text>Changing begining and ending columns to describe regression relationships ie changing X and Y NEEED TO CHECK WHETHER Y=6 means AUC is the comparison column THis will relate to which descriptor i am trying to relate<jupyter_code>numdescriptors = 6 # Y= 6 if AUC must be last Y cannot be 7
xrange =  np.arange(0,numdescriptors,1)
Y=numdescriptors
for X in xrange:    
    MLR=linear_model.LinearRegression()
    MLR.fit(train[train.columns.values[X:Y]],train[train.columns.values[Y]])
    # WE are going to train using the first 8 values

    # make predictions on test and train set 
    trainpred=MLR.predict(train[train.columns.values[X:Y]])
    # predict the outputs using the training dataset
    testpred=MLR.predict(test[train.columns.values[X:Y]])
    # predict using test dataset
    #make parity plot 
    maxlimit  = train[train.columns.values[Y]].max()
    print(maxlimit)
    plt.figure(figsize=(7,7))
    plt.xlim([0,maxlimit]);
    plt.ylim([0,maxlimit]);
    print('This is for X ='+str(X)+ '& Y ='+str(Y))
    plt.scatter(train[train.columns.values[Y]],trainpred, label='Training')
    plt.scatter(test[train.columns.values[Y]],testpred,color='r', label='Test')
    plt.plot([0,maxlimit],[0,maxlimit],lw=4,color='black')
    plt.legend()
    plt.xlabel('Actual Output')
    plt.ylabel('Predicted Output')

    #calculate the test and train error
    
    print("Train error",mean_squared_error(train[train.columns.values[Y]],trainpred)) # MSE of training dataset
    print("Test error",mean_squared_error(test[train.columns.values[Y]],testpred))
    # usually the training error is less than the test error<jupyter_output><empty_output><jupyter_text>Ridge & Lasso Regression<jupyter_code>from sklearn import linear_model
#from pandas.plotting import scatter_matrix
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import Ridge
from sklearn.preprocessing import StandardScaler
%matplotlib inline
#normalized data for Ridge / LASSO 
# train_normalized=train/train.std()
# test_normalized=test/test.std()
scaler = StandardScaler().fit(train[inputs])
train_normalized = pd.DataFrame(data=scaler.transform(train[inputs]), columns=train.columns[:-1])
test_normalized = pd.DataFrame(data=scaler.transform(test[inputs]), columns=test.columns[:-1])
# Columns cannot have strings because could not convert string to float: "['H2O']"
test_normalized.head() 
raman_ridge=Ridge()
numdescriptors = 7 # Y=6 for AUC to be last
yrange =  np.arange(1,numdescriptors,1)
for Y in yrange:
    a=1e1
    raman_ridge.set_params(alpha=a)
    raman_ridge.fit(train_normalized[train.columns.values[0:Y]],train_normalized[train.columns.values[Y]])
    print('This is for Y ='+str(Y))
    print("Train error",mean_squared_error(train_normalized[train.columns.values[Y]],raman_ridge.predict(
            train_normalized[train.columns.values[0:Y]])))

    print("Test error",mean_squared_error(test_normalized[train.columns.values[Y]],raman_ridge.predict(
            test_normalized[train.columns.values[0:Y]])))
    plt.figure(figsize=(7,7))
    plt.xlim([-2.5,2.5]);
    plt.ylim([-2.5,2.5]);
    plt.scatter(train_normalized[train.columns.values[Y]],raman_ridge.predict(
            train_normalized[train.columns.values[0:Y]]), label='Training')
    plt.scatter(test_normalized[train.columns.values[Y]],raman_ridge.predict(
            test_normalized[train.columns.values[0:Y]]),color='r', label='Test')
    maxlimit  = train[train.columns.values[Y]].max()
    minlimit  = train[train.columns.values[Y]].min()
    plt.plot([-2.5,2.5],[-2.5,2.5],lw=4,color='black')
    plt.legend()
    plt.xlabel('Actual Output')
    plt.ylabel('Predicted Output')
# RIDGE REGRESSION LOOKING AT RELATIONSHIPS OF ALL Descriptors vs. AUC I think!!
raman_ridge=Ridge()
numdescriptors = 7
yrange =  np.arange(1,numdescriptors,1)
for Y in yrange:
    a=1e1
    raman_ridge.set_params(alpha=a)
    raman_ridge.fit(train_normalized[train.columns.values[0:Y]],
                    train_normalized[train.columns.values[Y]])
    print('This is for Y ='+str(Y))
    print("Train error",mean_squared_error(train_normalized[train.columns.values[Y]],raman_ridge.predict(
            train_normalized[train.columns.values[0:Y]])))

    print("Test error",mean_squared_error(test_normalized[train.columns.values[Y]],raman_ridge.predict(
            test_normalized[train.columns.values[0:Y]])))
    plt.figure(figsize=(7,7))
    plt.xlim([-2.5,2.5]);
    plt.ylim([-2.5,2.5]);
    plt.scatter(train_normalized[train.columns.values[Y]],raman_ridge.predict(
            train_normalized[train.columns.values[0:Y]]), label='Training')
    plt.scatter(test_normalized[train.columns.values[Y]],raman_ridge.predict(
            test_normalized[train.columns.values[0:Y]]),color='r', label='Test')
    maxlimit  = train[train.columns.values[Y]].max()
    minlimit  = train[train.columns.values[Y]].min()
    plt.plot([-2.5,2.5],[-2.5,2.5],lw=4,color='black')
    plt.legend()
    plt.xlabel('Actual Output')
    plt.ylabel('Predicted Output')
raman_ridge=Ridge()
numdescriptors = 6 # Y=6 for AUC to be last
xrange =  np.arange(0,numdescriptors,1)
Y=numdescriptors
for X in xrange:
    a=1e1
    raman_ridge.set_params(alpha=a)
    raman_ridge.fit(train_normalized[train.columns.values[X:Y]],train_normalized[train.columns.values[Y]])
    print('This is for X ='+str(X)+'& Y =' + str(Y))
    print("Train error",mean_squared_error(train_normalized[train.columns.values[Y]],raman_ridge.predict(
            train_normalized[train.columns.values[X:Y]])))

    print("Test error",mean_squared_error(test_normalized[train.columns.values[Y]],raman_ridge.predict(
            test_normalized[train.columns.values[X:Y]])))
    plt.figure(figsize=(7,7))
    plt.xlim([-2.5,2.5]);
    plt.ylim([-2.5,2.5]);
    plt.scatter(train_normalized[train.columns.values[Y]],raman_ridge.predict(
            train_normalized[train.columns.values[X:Y]]), label='Training')
    plt.scatter(test_normalized[train.columns.values[Y]],raman_ridge.predict(
            test_normalized[train.columns.values[X:Y]]),color='r', label='Test')
    maxlimit  = train[train.columns.values[Y]].max()
    minlimit  = train[train.columns.values[Y]].min()
    plt.plot([-2.5,2.5],[-2.5,2.5],lw=4,color='black')
    plt.legend()
    plt.xlabel('Actual Output')
    plt.ylabel('Predicted Output')<jupyter_output><empty_output><jupyter_text>2-2 Example of searching the $\alpha$ space in RR<jupyter_code>train,test = train_test_split(dat, test_size=0.2, random_state=1000)
# RR vs lambda (based on sklearn tutorial)
coefs = []
trainerror = []
testerror = []

Y = numdescriptors
X = 0
# do you know what is happening here? 
lambdas = np.logspace(-6,6,200) # SEACH THE RANGE OF THE LAMBDA SPACE
model=Ridge()

# loop over lambda values (strength of regularization)
for l in lambdas:
    model.set_params(alpha=l)
    model.fit(train_normalized[train.columns.values[X:Y]],train_normalized[train.columns.values[Y]])
    coefs.append(model.coef_)
    trainerror.append(mean_squared_error(train_normalized[train.columns.values[Y]],model.predict(
        train_normalized[train.columns.values[X:Y]])))
    testerror.append(mean_squared_error(test_normalized[train.columns.values[Y]],model.predict(
        test_normalized[train.columns.values[X:Y]])))

# what is being plotted here? 
plt.figure(figsize=(10,3))
plt.subplot(121)
plt.plot(lambdas,coefs)
plt.xscale('log')
plt.xlabel('$\lambda$')
plt.ylabel('coefs')
plt.title('RR coefs vs $\lambda$')
plt.subplot(122)
plt.plot(lambdas,trainerror,label='train error')
plt.plot(lambdas,testerror,label='test error')
plt.xscale('log')
plt.xlabel('$\lambda$')
plt.ylabel('error')
plt.legend(loc=1)
plt.title('error vs $\lambda$')<jupyter_output><empty_output><jupyter_text>Part 3: LASSO regression  (same data as Part 1)* The lasso improves over ridge by also providing a variable selection tool!* The lasso minimizer is $RSS + \lambda \sum_{j=1}^{p}\lvert\beta_j\rvert$* Ridge regression does not set any of the coefficients exactly to zero but can shrink all of them- final model still includes all p predictors- Lasso is similar to ridge regression with a absolute value- It provides the possibility that some of the coeeficients can take a value of zero.- Like ri<jupyter_code># also based on sklearn tutorials
# what the hell is happening in this cell?
coefs = []
trainerror = []
testerror = []

lambdas = np.logspace(-6,6,200)
model=linear_model.Lasso()
numdescriptors = 6 # Y must be 6 for AUC to be last
Y = numdescriptors
X = 0
# loop over lambda values (strength of regularization)
for l in lambdas:
    model.set_params(alpha=l,max_iter=1e6)
    model.fit(train_normalized[train.columns.values[X:Y]],train_normalized[train.columns.values[Y]])
    coefs.append(model.coef_)
    trainerror.append(mean_squared_error(train_normalized[train.columns.values[Y]],model.predict(
        train_normalized[train.columns.values[X:Y]])))
    testerror.append(mean_squared_error(test_normalized[train.columns.values[Y]],model.predict(
        test_normalized[train.columns.values[X:Y]])))
yrange =  np.arange(1,7,1)
for Y in yrange:
    a=1e1
    model.set_params(alpha=a)
    model.fit(train_normalized[train.columns.values[X:Y]],train_normalized[train.columns.values[Y]])
    print('This is for Y ='+str(Y))
    print("Train error",trainerror[Y])
    print("Test error",testerror[Y])
    plt.figure(figsize=(7,7))
    plt.xlim([-2.5,2.5]);
    plt.ylim([-2.5,2.5]);
    plt.scatter(train_normalized[train.columns.values[Y]],model.predict(
        train_normalized[train.columns.values[X:Y]]), label='Training')
    plt.scatter(test_normalized[train.columns.values[Y]],model.predict(
        test_normalized[train.columns.values[X:Y]]),color='r', label='Test')
    maxlimit  = train[train.columns.values[Y]].max()
    minlimit  = train[train.columns.values[Y]].min()
    plt.plot([-2.5,2.5],[-2.5,2.5],lw=4,color='black')
    plt.legend()
    plt.xlabel('Actual Output')
    plt.ylabel('Predicted Output')
# also based on sklearn tutorials
# what the hell is happening in this cell?
coefs = []
trainerror = []
testerror = []

lambdas = np.logspace(-6,6,200)
model=linear_model.Lasso()
numdescriptors = 6 # Y must be 6 for AUC to be last
Y = numdescriptors
X = 0
# loop over lambda values (strength of regularization)
for l in lambdas:
    model.set_params(alpha=l,max_iter=1e6)
    model.fit(train_normalized[train.columns.values[X:Y]],train_normalized[train.columns.values[Y]])
    coefs.append(model.coef_)
    trainerror.append(mean_squared_error(train_normalized[train.columns.values[Y]],model.predict(
        train_normalized[train.columns.values[X:Y]])))
    testerror.append(mean_squared_error(test_normalized[train.columns.values[Y]],model.predict(
        test_normalized[train.columns.values[X:Y]])))
xrange =  np.arange(0,numdescriptors-1,1)
for X in xrange:
    a=1e1
    model.set_params(alpha=a)
    model.fit(train_normalized[train.columns.values[X:Y]],train_normalized[train.columns.values[Y]])
    print('This is for Y ='+str(Y))
    print("Train error",trainerror[Y])
    print("Test error",testerror[Y])
    plt.figure(figsize=(7,7))
    plt.xlim([-2.5,2.5]);
    plt.ylim([-2.5,2.5]);
    plt.scatter(train_normalized[train.columns.values[Y]],model.predict(
        train_normalized[train.columns.values[X:Y]]), label='Training')
    plt.scatter(test_normalized[train.columns.values[Y]],model.predict(
        test_normalized[train.columns.values[X:Y]]),color='r', label='Test')
    maxlimit  = train[train.columns.values[Y]].max()
    minlimit  = train[train.columns.values[Y]].min()
    plt.plot([-2.5,2.5],[-2.5,2.5],lw=4,color='black')
    plt.legend()
    plt.xlabel('Actual Output')
    plt.ylabel('Predicted Output')
plt.figure(figsize=(10,3))
plt.locator_params(nbins=5)
plt.subplot(121)
plt.plot(lambdas,coefs)
plt.xscale('log')
plt.xlabel('$\lambda$')
plt.ylabel('coefs')
plt.title('RR coefs vs $\lambda$')
#plt.xlim(1e-4,1e0)

plt.subplot(122)
plt.plot(lambdas,trainerror,label='train error')
plt.plot(lambdas,testerror,label='test error')
plt.xscale('log')
plt.xlabel('$\lambda$')
plt.ylabel('error')
#plt.xlim(1e-4,1e0)
#plt.ylim(0,0.5)
plt.legend(loc=1)
plt.title('error vs $\lambda$')<jupyter_output><empty_output><jupyter_text>WE SIMPLIFIED OUR MODEL BY GETTING RID OF THE RR COEFS NOT EQUAL TO ZERO AT ZERO LAMBDA. Other things to consider if you have more time * Note we did not scale the features in the MLR, try it out and verify the final error doesnt' change!* Make sure you undersand how to make _predictions_ with supervised learning models that are trained on scaled/normalized data* Plot the residuals and verify if errors are distributed normally* Make a parity plot including the predictions from ridge and LASSO * Compare errors between all three * Explore the effect of training/testing split * Look at the shrinkage/regularization situation when predicting Y2 vs Y1...  Trying to see if any of these descriptors  ['fraction','sigma','center','amplitude','fwhm','height','auc'] can relate to the peakidentified labels  Support Vector Machine<jupyter_code>inputs=['fraction','sigma','center','amplitude','fwhm','height','auc']
X=dat[inputs]
y = dat['labels']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=1010)<jupyter_output><empty_output><jupyter_text>Linear SVM<jupyter_code>#Import svm model
from sklearn import svm
from sklearn import metrics

#Create a svm Classifier
clf = svm.SVC(kernel='linear') # Linear Kernel

#Train the model using the training sets
clf.fit(X_train, y_train)

#Predict the response for test dataset
y_pred = clf.predict(X_test)
from sklearn.metrics import classification_report, confusion_matrix  
print(confusion_matrix(y_test, y_pred))  
print(classification_report(y_test, y_pred))  
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))
# # Model Precision: what percentage of positive tuples are labeled as such?
# print("Precision:",metrics.precision_score(y_test, y_pred),average='macro')

# # Model Recall: what percentage of positive tuples are labelled as such?
# print("Recall:",metrics.recall_score(y_test, y_pred), average='macro')<jupyter_output><empty_output><jupyter_text>Polynomial kernal (https://stackabuse.com/implementing-svm-and-kernel-svm-with-pythons-scikit-learn/)<jupyter_code>from sklearn.svm import SVC  
svclassifier = SVC(kernel='poly', degree=8)  
svclassifier.fit(X_train, y_train)  
y_pred = svclassifier.predict(X_test)  
from sklearn.metrics import classification_report, confusion_matrix  
print(confusion_matrix(y_test, y_pred))  
print(classification_report(y_test, y_pred))  
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))<jupyter_output><empty_output><jupyter_text>Gaussian SVM<jupyter_code>from sklearn.svm import SVC  
svclassifier = SVC(kernel='rbf')  
svclassifier.fit(X_train, y_train)  
y_pred = svclassifier.predict(X_test)  
print(confusion_matrix(y_test, y_pred))  
print(classification_report(y_test, y_pred))  
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))<jupyter_output><empty_output><jupyter_text>Sigmoid SVM<jupyter_code>from sklearn.svm import SVC  
svclassifier = SVC(kernel='sigmoid')  
svclassifier.fit(X_train, y_train)  
y_pred = svclassifier.predict(X_test)  
y_pred = svclassifier.predict(X_test)  
print(confusion_matrix(y_test, y_pred))  
print(classification_report(y_test, y_pred))  
print("Accuracy:",metrics.accuracy_score(y_test, y_pred))<jupyter_output><empty_output><jupyter_text>Decision Trees  specifically to see if below descriptors can find and predict Area under curve  Strings needed to be converted to floats anyways so the labelling relationship was not possible<jupyter_code>from sklearn import linear_model
from pandas.plotting import scatter_matrix
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import Ridge
from sklearn import tree
DT=tree.DecisionTreeRegressor(max_depth=3)
DT.fit(train[['fraction','sigma','center','amplitude','fwhm','height']],train.auc)
import os
#conda install -c conda-forge pydotplus=2.0.2
# http://www.webgraphviz.com
import pydotplus 
with open("basic_tree.dot", 'w') as f:
    f = tree.export_graphviz(DT, out_file=f)
#http://www.webgraphviz.com
# os.unlink('basic_tree.dot')

dot_data = tree.export_graphviz(DT, out_file=None,feature_names=['fraction','sigma','center','amplitude','fwhm','height'])
graph = pydotplus.graph_from_dot_data(dot_data) 
#graph.write_pdf("basic_tree.pdf") 
os.environ["PATH"] += os.pathsep + 'C:/Users/koolk/Anaconda3/Library/bin/graphviz/'
from IPython.display import SVG, display
display(SVG(graph.create_svg()))
# make predictions on test and train set 
trainpred=DT.predict(train[['fraction','sigma','center','amplitude','fwhm','height']])
testpred=DT.predict(test[['fraction','sigma','center','amplitude','fwhm','height']])

maxlimit =testpred.max()
print(maxlimit)
#parity plot 
plt.figure(figsize=(4,4))
plt.xlim([0,maxlimit]);
plt.ylim([0,maxlimit]);
plt.scatter(train.auc,trainpred, label ='train')
plt.scatter(test.auc,testpred,color='r', label = 'test')
plt.plot([0,maxlimit],[0,maxlimit],lw=4,color='black')
plt.legend()
#calculate the test and train error
print("Train error",mean_squared_error(train.auc,trainpred))
print("Test error",mean_squared_error(test.auc,testpred))
trainerror = []
testerror = []

trees = np.arange(1,20,1)
#model=tree.DecisionTreeRegressor()

# loop over lambda values (strength of regularization)
for t in trees:
    model=tree.DecisionTreeRegressor(max_depth=t)
    model.fit(train[['fraction','sigma','center','amplitude','fwhm','height']],train.auc)
    trainerror.append(mean_squared_error(train.auc,model.predict(
        train[['fraction','sigma','center','amplitude','fwhm','height']])))
    testerror.append(mean_squared_error(test.auc,model.predict(
        test[['fraction','sigma','center','amplitude','fwhm','height']])))
plt.figure(figsize=(8,4))
plt.subplot(121)
plt.plot(trees,trainerror,marker='o',label='testerror')
plt.plot(trees,testerror,marker="s",label='trainerror')
plt.legend()
plt.xlabel('Max tree depth')
plt.ylabel('MSE for $auc$')
plt.subplot(122)
plt.plot(trees,trainerror,marker='o',label='testerror')
plt.plot(trees,testerror,marker="s",label='trainerror')
plt.ylim((0,1))
plt.xlim((5,15))
plt.legend()
plt.xlabel('Max tree depth')
plt.ylabel('MSE for $auc$')<jupyter_output><empty_output><jupyter_text>SVM from lecture<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split

from matplotlib.colors import ListedColormap
X, y = make_classification(n_features=2, n_redundant=0, n_informative=2,
                           random_state=1, n_clusters_per_class=1, class_sep=0.7)
rng = np.random.RandomState(2)
#X += 2 * rng.uniform(size=X.shape)
data = (X, y)
X_train, X_test, y_train, y_test = \
        train_test_split(X, y, test_size=.4, random_state=42)
figure, ax = plt.subplots(figsize=(4,4))

cm = plt.cm.RdBu
cm_bright = ListedColormap(['#FF0000', '#0000FF'])
# Plot the training points
ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright,
           edgecolors='k', label='Training')
# Plot the testing points
ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6,
           edgecolors='k', label='Test')
#ax.set_xlim([-3,4])
#ax.set_ylim([0,4])
ax.legend()
from sklearn.svm import SVC
clf = SVC(kernel='linear', C=1)

h=0.02
x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

clf.fit(X_train, y_train)
score = clf.score(X_test, y_test)

figure, ax = plt.subplots(figsize=(4,4))
# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
if hasattr(clf, "decision_function"):
    Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
else:
    Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]

# Put the result into a color plot
Z = Z.reshape(xx.shape)
Zbi = Z > np.median(Z)
ax.contourf(xx, yy, Zbi, cmap=cm, alpha=.1)

# Plot the training points
ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright,
           edgecolors='k', label='Training')
# Plot the testing points
ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright,
           edgecolors='k', alpha=0.6, label='Test')

ax.legend()
X, y = make_classification(n_features=2, n_redundant=0, n_informative=2,
                           random_state=1, n_clusters_per_class=1, class_sep=0.8)
rng = np.random.RandomState(2)
X += 2 * rng.uniform(size=X.shape)
data = (X, y)

X_train, X_test, y_train, y_test = \
        train_test_split(X, y, test_size=.4, random_state=42)
figure, ax = plt.subplots(figsize=(4,4))

cm = plt.cm.RdBu
cm_bright = ListedColormap(['#FF0000', '#0000FF'])
# Plot the training points
ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright,
           edgecolors='k', label='Training')
# Plot the testing points
ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright, alpha=0.6,
           edgecolors='k', label='Test')
#ax.set_xlim([-3,4])
#ax.set_ylim([0,4])
ax.legend()
C_values = [0.00000001, 0.001, 0.1, 1, 10, 100, 1000, 10000]
figure, axes = plt.subplots(nrows=2, ncols=4, figsize=(12, 9))

axes = [item for sublist in axes for item in sublist]

h=0.02
x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

for C, ax in zip(C_values, axes):
    clf = SVC(kernel='linear', C=C)
    clf.fit(X_train, y_train)
    score = clf.score(X_test, y_test)

    # Plot the decision boundary. For that, we will assign a color to each
    # point in the mesh [x_min, x_max]x[y_min, y_max].
    if hasattr(clf, "decision_function"):
        Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
    else:
        Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]

    # Put the result into a color plot
    Z = Z.reshape(xx.shape)
    Zbi = Z > np.median(Z)
    ax.contourf(xx, yy, Zbi, cmap=cm, alpha=.1)

    # Plot the training points
    ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright,
               edgecolors='k', label='Training')
    # Plot the testing points
#     ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright,
#                edgecolors='k', alpha=0.6, label='Test')

    ax.legend()
    ax.set_title('C Value: {}'.format(str(C)))
from sklearn.datasets import make_circles

data = make_circles(noise=0.2, factor=0.5, random_state=1)
X, y = data

X_train, X_test, y_train, y_test = \
        train_test_split(X, y, test_size=.4, random_state=42)
clf = SVC(kernel='linear', C=1)

h=0.02
x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

clf.fit(X_train, y_train)
score = clf.score(X_test, y_test)

figure, ax = plt.subplots(figsize=(4,4))
# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
if hasattr(clf, "decision_function"):
    Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
else:
    Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]

# Put the result into a color plot
Z = Z.reshape(xx.shape)
Zbi = Z > np.median(Z)
ax.contourf(xx, yy, Zbi, cmap=cm, alpha=.1)

# Plot the training points
ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright,
           edgecolors='k', label='Training')
# Plot the testing points
ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright,
           edgecolors='k', alpha=0.6, label='Test')

ax.legend()
clf = SVC(gamma=4, C=0.1)

h=0.02
x_min, x_max = X[:, 0].min() - .5, X[:, 0].max() + .5
y_min, y_max = X[:, 1].min() - .5, X[:, 1].max() + .5
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
                     np.arange(y_min, y_max, h))

clf.fit(X_train, y_train)
score = clf.score(X_test, y_test)

figure, ax = plt.subplots(figsize=(4,4))
# Plot the decision boundary. For that, we will assign a color to each
# point in the mesh [x_min, x_max]x[y_min, y_max].
if hasattr(clf, "decision_function"):
    Z = clf.decision_function(np.c_[xx.ravel(), yy.ravel()])
else:
    Z = clf.predict_proba(np.c_[xx.ravel(), yy.ravel()])[:, 1]

# Put the result into a color plot
Z = Z.reshape(xx.shape)
Zbi = Z > np.median(Z)
ax.contourf(xx, yy, Zbi, cmap=cm, alpha=.1)

# Plot the training points
ax.scatter(X_train[:, 0], X_train[:, 1], c=y_train, cmap=cm_bright,
           edgecolors='k', label='Training')
# Plot the testing points
ax.scatter(X_test[:, 0], X_test[:, 1], c=y_test, cmap=cm_bright,
           edgecolors='k', alpha=0.6, label='Test')

ax.legend()<jupyter_output><empty_output><jupyter_text>Cross Validation from NN<jupyter_code>from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
seed = 21899
X = df[['mass', 'voc', 'jsc', 'e_homo_alpha', 'e_gap_alpha', 
        'e_lumo_alpha']].values
Y = df[['pce']].values
X_train_pn, X_test_pn, y_train, y_test = train_test_split(X, Y,
                                                    test_size=0.20,
                                                    random_state=seed)
# create the scaler from the training data only and keep it for later use
X_train_scaler = StandardScaler().fit(X_train_pn)
# apply the scaler transform to the training data
X_train = X_train_scaler.transform(X_train_pn)
X_test = X_train_scaler.transform(X_test_pn)
test_loss = estimator.model.evaluate(X_test, y_test)
print("test set mse is %.2f" % test_loss)
kfold = KFold(n_splits=10, random_state=seed)
results = cross_val_score(estimator, X_train, y_train, cv=kfold)
print("Results: %.2f (%.2f) MSE" % (-1 * results.mean(), results.std()))
estimators = []
estimators.append(('standardize', StandardScaler()))
estimators.append(('mlp', KerasRegressor(build_fn=simple_model,
        epochs=150, batch_size=25000, verbose=0)))
pipeline = Pipeline(estimators)
kfold = KFold(n_splits=10, random_state=seed)
results = cross_val_score(pipeline, X_train, y_train, cv=kfold)
print('MSE mean: %.4f ; std: %.4f' % (-1 * results.mean(), results.std()))<jupyter_output><empty_output><jupyter_text>Validation<jupyter_code>from sklearn.model_selection import train_test_split
from sklearn.linear_model import LinearRegression
x = np.linspace(0, 10, 1000)
y = 0.2*x + 1*x**2 - 0.3*x**3 + 0.021*x**4 + 0.3*np.random.rand(1000)*x

plt.scatter(x, y, alpha=0.1)

X = pd.DataFrame({'x':x, 'x2':x**2, 'x3':x**3, 'x4':x**4, 'x5':x**5, 'x6':x**6, 'x7':x**7, 'x8':x**8,
                  'x9':x**9, 'x10':x**10, 'x11':x**12, 'x13':x**13, 'x14':x**14, 'x15':x**15, 'x16':x**16, 'x17':x**17})
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)
reg = LinearRegression().fit(X_train[['x']], y_train) #Simple LINEAR REGRESSION with multiple polynomial 
#PUT IN ARRAYS OF ONE DATASET OR DATASET OF INDEPENDENT FEATURES
y_pred = reg.predict(X_test[['x']])
plt.scatter(x, y, alpha=0.1)
plt.plot(X_test[['x']], y_pred, linewidth=2)
from sklearn.metrics import mean_squared_error
mean_squared_error(y_test, y_pred)
fig, ax = plt.subplots(figsize=(8,6))
ax.scatter(x, y, alpha=0.1)
mse = np.zeros(17)

for i in range(1, 17):
    reg = LinearRegression().fit(X_train[X_train.columns[0:i].values], y_train)
    y_pred = reg.predict(X_test[X_test.columns[0:i].values])

    ax.scatter(X_test[['x']], y_pred, marker='o', s=25, label=i)
    mse[i-1] = mean_squared_error(y_test, y_pred)
    ax.legend()
fig, ax = plt.subplots(figsize=(8,6))
ax.plot(np.linspace(1, 16, 17), mse, linewidth=4)
ax.set_ylabel('Mean Squared Error')
ax.set_xlabel('Degree of Polynomial')
fig, ax = plt.subplots(figsize=(8,6))

for run in range(8):
    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33)
    mse = np.zeros(17)

    for i in range(1, 18):
        reg = LinearRegression().fit(X_train[X_train.columns[0:i].values], y_train)
        y_pred = reg.predict(X_test[X_test.columns[0:i].values])

        mse[i-1] = mean_squared_error(y_test, y_pred)

    
    ax.plot(np.linspace(1, 16, 17), mse, linewidth=4, color='b', alpha=0.2)
    ax.set_ylabel('Mean Squared Error')
    ax.set_xlabel('Degree of Polynomial')<jupyter_output><empty_output><jupyter_text>Kfold validation<jupyter_code>from sklearn.model_selection import KFold
kf = KFold(n_splits=10, shuffle=True)
kf.get_n_splits(X, y)
kf.split(X,y) # now is an iterator
kf = KFold(n_splits=10, shuffle=True)
kf.get_n_splits(X, y)

fig, ax = plt.subplots(figsize=(8,6))

mse = np.zeros((17, 10))
j = 0
for train_index, test_index in kf.split(X, y):
    #print("TRAIN:", train_index, "TEST:", test_index)
    X_train, X_test = X.iloc[train_index], X.iloc[test_index]
    y_train, y_test = y[train_index], y[test_index]

    for i in range(1, 18):
        reg = LinearRegression().fit(X_train[X_train.columns[0:i].values], y_train)
        y_pred = reg.predict(X_test[X_test.columns[0:i].values])

        mse[i-1, j] = mean_squared_error(y_test, y_pred)

    
    ax.plot(np.linspace(1, 16, 17), mse[:, j], linewidth=4, color='b', alpha=0.09)
    ax.set_ylabel('Mean Squared Error')
    ax.set_xlabel('Degree of Polynomial')
    j += 1
    
avg_mse = mse.mean(axis=1)
ax.plot(np.linspace(1, 16, 17), avg_mse, color='purple')
avg_mse
df = pd.read_excel('../examples/quad.xlsx')<jupyter_output><empty_output><jupyter_text>Neural Networks<jupyter_code>from keras.models import Sequential
from keras.layers import Dense
from keras.wrappers.scikit_learn import KerasRegressor

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
test_loss = estimator.model.evaluate(X_test, y_test)
print("test set mse is %.2f" % test_loss)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1531930699636518, 'fraction_numerical': 0.019341487357724656, 'mean_word_length': 5.119824646858256, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 85, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '20', 'dir': 'jupyter-structured-clean-dedup', 'id': '396809', 'license': 'MIT', 'n_tokens_mistral': 14471, 'n_tokens_neox': 13865, 'n_words': 2888, 'path': 'development/MLdev.ipynb', 'repo_name': 'kernb2/raman-spectra-decomp-analysis', 'size': '37691'}