<jupyter_start><jupyter_text>**Section 1: Wine Dataset** **Task 1: Load Data and Task 2: Splitting the data**<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_wine
dataset = load_wine()
from sklearn.model_selection import train_test_split
X_train, X_test,y_train,y_test= train_test_split(dataset.data , dataset.target, random_state  = 2810)<jupyter_output><empty_output><jupyter_text>**Task 3: Cross Validation**<jupyter_code>from sklearn.model_selection import cross_val_score
from sklearn.svm import SVC
svc = SVC()
print(cross_val_score(svc,X_train, y_train))
print('The Generalisation accuracy is',np.mean(cross_val_score(svc,X_train, y_train)))
print('The Cross Val error is',np.mean(1-cross_val_score(svc,X_train, y_train)))<jupyter_output>The Cross Val error is 0.30769230769230765<jupyter_text>**Task 4: Test Error rate**<jupyter_code>svc.fit(X_train,y_train)
print('Test Error rate',1-svc.score(X_test,y_test))<jupyter_output>Test Error rate 0.37777777777777777<jupyter_text>**The Performance on the Test-set is worse than the performance on Cross_validation sets, this is largely due to small size of training set** **Task 5:Creating Pipeline with different normalisation and Task 6: Predicting Labels and calculating Test-error rate**<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.preprocessing import MinMaxScaler
pipeline = Pipeline([('scaler',MinMaxScaler()), ('svm',SVC())])
from sklearn.model_selection import GridSearchCV
param_grid ={'svm__C':[0.001,0.01,0.1,1,10,100], 'svm__gamma':[0.001,0.01,0.1,1,10,100]}
grid = GridSearchCV(pipeline, param_grid = param_grid, cv =5, n_jobs = -1)
grid.fit(X_train,y_train)
y_pred = grid.predict(X_test)
print('Best Cross_validation accuracy:', grid.best_score_)
print('Test set score:', grid.score(X_test,y_test))
print('Best Parameters:', grid.best_params_)
print('The Test Error rate-',np.mean(y_pred != y_test))
from sklearn.preprocessing import StandardScaler
pipeline = Pipeline([('scaler',StandardScaler()), ('svm',SVC())])
grid = GridSearchCV(pipeline, param_grid = param_grid, cv =5, n_jobs = -1)
grid.fit(X_train,y_train)
y_pred = grid.predict(X_test)
print('Best Cross_validation accuracy:', grid.best_score_)
print('Test set score:', grid.score(X_test,y_test))
print('Best Parameters:', grid.best_params_)
print('The Test Error rate-',np.mean(y_pred != y_test))
from sklearn.preprocessing import Normalizer
pipeline = Pipeline([('scaler',Normalizer()), ('svm',SVC())])
grid = GridSearchCV(pipeline, param_grid = param_grid, cv =5, n_jobs = -1)
grid.fit(X_train,y_train)
y_pred = grid.predict(X_test)
print('Best Cross_validation accuracy:', grid.best_score_)
print('Test set score:', grid.score(X_test,y_test))
print('Best Parameters:', grid.best_params_)
print('The Test Error rate-',np.mean(y_pred != y_test))
from sklearn.preprocessing import RobustScaler
pipeline = Pipeline([('scaler',RobustScaler()), ('svm',SVC())])
grid = GridSearchCV(pipeline, param_grid = param_grid, cv =5, n_jobs = -1)
grid.fit(X_train,y_train)
y_pred = grid.predict(X_test)
print('Best Cross_validation accuracy:', grid.best_score_)
print('Test set score:', grid.score(X_test,y_test))
print('Best Parameters:', grid.best_params_)
print('The Test Error rate-',np.mean(y_pred != y_test))<jupyter_output>Best Cross_validation accuracy: 0.9925925925925926
Test set score: 0.9555555555555556
Best Parameters: {'svm__C': 1, 'svm__gamma': 0.01}
The Test Error rate- 0.044444444444444446<jupyter_text>**For Wine Dataset, MinMax appears to be best way to normalise as it is providing the best Cross-Validation accuracy and best Test set score. However,the difference between the performance MinMax and Standard Normalisation is not that big and both of them provide comparable results.Robust Scaler gives test-error rate equal to MinMax Scaler but it has lower Cross-validation accuracy. For the next task, I am going to use MinMax to preprocess data.** **Task 7: Implementing Cross-Conformal Predictors** '''For the implementaion of cross-conformal predictors, I have used CV = 5. I have calculated ranks for each caliberation fold and have added the rank for the other caliberation fold using a loop. Since, I have not created an augmented dataset, the way I have found the rank of each test sample for each caliberation set does not require me to subtract one from the rank as this way if my test sample for a given fold is the strangest, the rank will be zero already and hence no need to subtract 1.'''<jupyter_code>pipeline = Pipeline([('scaler',MinMaxScaler()), ('svm',SVC())]) #Redefining Pipeline
from sklearn.model_selection import KFold
kf = KFold(shuffle = True, random_state = 2810)
grid1 = GridSearchCV(pipeline, param_grid = param_grid, cv =5, n_jobs = -1)
fold_rank = np.zeros((len(X_test),3))
for rest_index, fold_index in kf.split(X_train):
    X_rest,X_fold = X_train[rest_index], X_train[fold_index]
    y_rest,y_fold = y_train[rest_index], y_train[fold_index]
    conformity_fold = np.zeros(len(y_fold))
    grid1.fit(X_rest,y_rest)
    conformity_scores  = grid1.decision_function(X_fold) 
    test_decision = grid1.decision_function(X_test)
    for i in range(len(y_fold)):
        conformity_fold[i] = conformity_scores[i,y_fold[i]]
    for j in range(len(y_test)):
        for k in range(3):
            fold_rank[j,k] = fold_rank[j,k]+np.sum(test_decision[j,k] >= conformity_fold)
p_value = (fold_rank+1)/(len(X_train)+1)
labels = np.unique(y_train)
pred_targets = np.zeros(len(X_test))
for i in range(len(p_value)):
    pred_targets[i] = labels[np.argmax(p_value[i])]
print('The Test Error-',np.mean(pred_targets != y_test))
%matplotlib inline
import matplotlib.pyplot as plt
eps = np.zeros(100)
err = np.zeros(100)
for k in range(100):
    eps[k] = k/100
    err[k] = 0
    for j in range(len(y_test)):
        if(p_value[j,y_test[j]]<= eps[k]):
            err[k] = err[k]+1
    err[k] = err[k]/len(y_test)
plt.plot(eps,err)
total = 0    
for i in range(len(y_test)):
    k = y_test[i]
    for j in range(len(labels)):
        if k!= j:
            total =total+ p_value[i,j]
avg_false_p = total/(len(p_value)*(len(labels)-1))
print('Avg False P-Value is' ,avg_false_p)<jupyter_output>Avg False P-Value is 0.008706467661691555<jupyter_text>**Section 2: USPS Dataset** **Task 1: Load Data and Task 2: Splitting the data**<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
data_test = np.genfromtxt('/home/smith/Downloads/zip.test')
data_train = np.genfromtxt('/home/smith/Downloads/zip.train')
dataset = np.concatenate([data_train, data_test]) #merging the data
from sklearn.model_selection import train_test_split
X_train, X_test,y_train,y_test= train_test_split(dataset[:,1:] , dataset[:,0], random_state  = 2810)<jupyter_output><empty_output><jupyter_text>**Task 3:Cross- Validation**<jupyter_code>from sklearn.model_selection import cross_val_score
from sklearn.svm import SVC
svc = SVC()
print(cross_val_score(svc,X_train, y_train))
print('The Generalisation accuracy is',np.mean(cross_val_score(svc,X_train, y_train)))
print('The Cross Val error is',np.mean(1-cross_val_score(svc,X_train, y_train)))<jupyter_output>The Cross Val error is 0.028968287026323793<jupyter_text>**Task 4:Test error rate**<jupyter_code>svc.fit(X_train,y_train)
print(1-svc.score(X_test,y_test))<jupyter_output>0.02537634408602152<jupyter_text>**The Performance on the Test-set is similar to the performance on Cross_validation sets due to large size of training set** **Task 5:Creating Pipeline with different normalisation and Task 6: Predicting Labels and Test error_rate**<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.preprocessing import MinMaxScaler
pipeline = Pipeline([('scaler',MinMaxScaler()), ('svm',SVC())])
from sklearn.model_selection import GridSearchCV
param_grid ={'svm__C':[0.001,0.01,0.1,1,10,100], 'svm__gamma':[0.001,0.01,0.1,1,10,100]}
grid = GridSearchCV(pipeline, param_grid = param_grid, cv =5, n_jobs = -1)
grid.fit(X_train,y_train)
y_pred = grid.predict(X_test)
print('Best Cross_validation accuracy:', grid.best_score_)
print('Test set score:', grid.score(X_test,y_test))
print('Best Parameters:', grid.best_params_)
print('The Test Error rate-',np.mean(y_pred != y_test))
from sklearn.preprocessing import StandardScaler
pipeline = Pipeline([('scaler',StandardScaler()), ('svm',SVC())])
grid = GridSearchCV(pipeline, param_grid = param_grid, cv =5, n_jobs = -1)
grid.fit(X_train,y_train)
y_pred = grid.predict(X_test)
print('Best Cross_validation accuracy:', grid.best_score_)
print('Test set score:', grid.score(X_test,y_test))
print('Best Parameters:', grid.best_params_)
print('The Test Error rate-',np.mean(y_pred != y_test))
from sklearn.preprocessing import RobustScaler
pipeline = Pipeline([('scaler',RobustScaler()), ('svm',SVC())])
grid = GridSearchCV(pipeline, param_grid = param_grid, cv =5, n_jobs = -1)
grid.fit(X_train,y_train)
y_pred = grid.predict(X_test)
print('Best Cross_validation accuracy:', grid.best_score_)
print('Test set score:', grid.score(X_test,y_test))
print('Best Parameters:', grid.best_params_)
print('The Test Error rate-',np.mean(y_pred != y_test))
from sklearn.preprocessing import Normalizer
pipeline = Pipeline([('scaler',Normalizer()), ('svm',SVC())])
grid = GridSearchCV(pipeline, param_grid = param_grid, cv =5, n_jobs = -1)
grid.fit(X_train,y_train)
y_pred = grid.predict(X_test)
print('Best Cross_validation accuracy:', grid.best_score_)
print('Test set score:', grid.score(X_test,y_test))
print('Best Parameters:', grid.best_params_)
print('The Test Error rate-',np.mean(y_pred != y_test))<jupyter_output>Best Cross_validation accuracy: 0.9731825591500696
Test set score: 0.978494623655914
Best Parameters: {'svm__C': 10, 'svm__gamma': 1}
The Test Error rate- 0.021505376344086023<jupyter_text>**For USPS Dataset, Normaliser appears to be best way to normalise as it is providing the best Cross-Validation accuracy and best Test set score.Since, the USPS dataset contains images, it is required to standardize the brightness, or the brightness and contrast,for each image separately and thus we use Normaliser class for per-sample normalisation. For the next task, I am going to use Normalizer method to preprocess data** **Task 7: Implementing Cross-Conformal Predictors**<jupyter_code>pipeline = Pipeline([('scaler',Normalizer()), ('svm',SVC())]) #Redefining Pipeline
from sklearn.model_selection import KFold
kf = KFold(shuffle = True, random_state = 2810)
grid1 = GridSearchCV(pipeline, param_grid = param_grid, cv =5, n_jobs = -1)
fold_rank = np.zeros((len(X_test),10))
for rest_index, fold_index in kf.split(X_train):
    X_rest,X_fold = X_train[rest_index], X_train[fold_index]
    y_rest,y_fold = y_train[rest_index], y_train[fold_index]
    conformity_fold = np.zeros(len(y_fold))
    grid1.fit(X_rest,y_rest)
    conformity_scores  = grid1.decision_function(X_fold) 
    test_decision = grid1.decision_function(X_test)
    for i in range(len(y_fold)):
        conformity_fold[i] = conformity_scores[i,int(y_fold[i])]
    for j in range(len(y_test)):
        for k in range(10):
            fold_rank[j,k] = fold_rank[j,k]+np.sum(test_decision[j,k] >= conformity_fold) 
p_value = (fold_rank+1)/(len(X_train)+1)
labels = np.unique(y_train)
pred_targets = np.zeros(len(X_test))
for i in range(len(p_value)):
    pred_targets[i] = labels[np.argmax(p_value[i])]
print('The Test error rate is',np.mean(pred_targets != y_test))
%matplotlib inline
import matplotlib.pyplot as plt
eps = np.zeros(100)
err = np.zeros(100)
for k in range(100):
    eps[k] = k/100
    err[k] = 0
    for j in range(len(y_test)):
        if(p_value[j,int(y_test[j])]<= eps[k]):
            err[k] = err[k]+1
    err[k] = err[k]/len(y_test)
plt.plot(eps,err)
total = 0    
for i in range(len(y_test)):
    k = y_test[i]
    for j in range(len(labels)):
        if k!= j:
            total =total+ p_value[i,j]
avg_false_p = total/(len(p_value)*(len(labels)-1))
print('Average False P-value is ',avg_false_p)<jupyter_output>Average False P-value is  0.004598339146014124
{'fraction_non_alphanumeric': 0.1305933046072257, 'fraction_numerical': 0.03256546237984753, 'mean_word_length': 6.3056900726392255, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 3, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '285652', 'license': 'MIT', 'n_tokens_mistral': 4537, 'n_tokens_neox': 4286, 'n_words': 1052, 'path': 'Cross Conformal Predictors with SVM.ipynb', 'repo_name': 'eshuenglish/Implementation-of-Conformal-Predictors', 'size': '12068'}