<jupyter_start><jupyter_text>Create classifiers<jupyter_code>rand_state=45
max_iter=1000000
# n_split for linear = 10 and for polynomial = 6
n_split_linear=10
n_split_poly=6

cv1 = StratifiedShuffleSplit(n_splits=n_split_linear, random_state=rand_state) 
cv2 = StratifiedShuffleSplit(n_splits=n_split_poly, random_state=rand_state) 
le = LabelEncoder()<jupyter_output><empty_output><jupyter_text>Question 1 and 2Question 1): Train a  linear multi-class classification SVM with no kernel.  Specify i) Your mapping function and ii) Your loss function  (20 points)Question 2) Describe a method to estimate your performance using an empirical method. Compare this estimate with a well known theoretical bound. Explain why/if there is a difference. (5 points)  Tinkering<jupyter_code># n_split for linear = 10 and for polynomial = 6
n_split_linear=10
n_split_poly=6

cv1 = StratifiedShuffleSplit(n_splits=n_split_linear, random_state=rand_state) 
cv2 = StratifiedShuffleSplit(n_splits=n_split_poly, random_state=rand_state) 
cv3 = StratifiedShuffleSplit(n_splits=3, random_state=rand_state) 
max_iter=10000000

lsvc_l1 = LinearSVC(penalty='l1', loss='squared_hinge', dual=False, max_iter=max_iter)
lsvc_l2 = LinearSVC(penalty='l2', loss='squared_hinge', dual=False, max_iter=max_iter)
lsvc_l3 = LinearSVC(penalty='l1', loss='squared_hinge', max_iter=max_iter)
lsvc_l4 = LinearSVC(penalty='l2', loss='squared_hinge', max_iter=max_iter)
lsvc_l5 = LinearSVC(penalty='l1', loss='hinge', max_iter=max_iter)
lsvc_l6 = LinearSVC(penalty='l2', loss='hinge', max_iter=max_iter)


# bsvc = SVC(kernel='linear', C=8, random_state=rand_state, max_iter=max_iter, class_weight='balanced')

# lsvc1 = LinearSVC(loss='hinge', max_iter=max_iter) # took forever with cv1, 
lsvc2 = LinearSVC(loss='squared_hinge', max_iter=max_iter, tol=1e-3)
lsvc3 = make_pipeline(StandardScaler(), LinearSVC(random_state=rand_state, max_iter=max_iter))
lsvc4 = make_pipeline(StandardScaler(), SVC(kernel='linear', random_state=rand_state, max_iter=max_iter))          

ovrsvc = OneVsRestClassifier(bsvc)

# Do NOT run again


ovrsvc_scores    = cross_val_score(ovrsvc, X, Y, cv=cv1, n_jobs=1)
print(ovrsvc_scores)
print(np.mean(ovrsvc_scores))
# Do NOT run again
# bsvc = SVC(kernel='linear', C=8, random_state=rand_state, max_iter=max_iter, class_weight='balanced')
bsvc = make_pipeline(StandardScaler(), SVC(kernel='linear', random_state=rand_state, \
                                           C=8, max_iter=max_iter, class_weight='balanced'))

bsvc_scores     = cross_val_score(bsvc, X, Y, cv=cv1, n_jobs=1)
print('one vs one')
print(bsvc_scores)
print(np.mean(bsvc_scores))


ovrsvc = OneVsRestClassifier(bsvc)
bsvc_scores     = cross_val_score(ovrsvc, X, Y, cv=cv1, n_jobs=1)
print('one vs rest')
print(bsvc_scores)
print(np.mean(bsvc_scores))


# lsvc3 = make_pipeline(StandardScaler(), LinearSVC(random_state=rand_state, max_iter=max_iter))
lsvc3 = make_pipeline(StandardScaler(), LinearSVC(random_state=rand_state, C=8, \
                                                  max_iter=max_iter, class_weight='balanced'))

lsvc3_scores     = cross_val_score(lsvc3, X, Y, cv=cv1, n_jobs=1)
print(lsvc3_scores)
print(np.mean(lsvc3_scores))
# Do NOT run again
# lsvc1 = LinearSVC(loss='hinge',C=8, random_state=rand_state, max_iter=max_iter, class_weight='balanced', tol=1e4) # took forever with cv1, 

lsvc1 = make_pipeline(StandardScaler(), LinearSVC(loss='squared_hinge', random_state=rand_state,\
                                                  C=8, max_iter=max_iter, class_weight='balanced'))


lsvc1_scores     = cross_val_score(lsvc1, X, Y, cv=cv1, n_jobs=1)
print('one vs rest')
print(lsvc1_scores)
print(np.mean(lsvc1_scores))

# Do NOT run again
lsvc2_scores     = cross_val_score(lsvc2, X, Y, cv=cv3, n_jobs=1)
print(lsvc2_scores)
print(np.mean(lsvc2_scores))
# Do NOT run again
lsvc3_scores     = cross_val_score(lsvc3, X, Y, cv=cv1, n_jobs=1)
print(lsvc3_scores)
print(np.mean(lsvc3_scores))
# Do NOT run again
lsvc4_scores     = cross_val_score(lsvc4, X, Y, cv=cv1, n_jobs=1)
print(lsvc4_scores)
print(np.mean(lsvc4_scores))
# Do NOT run again
lsvc_l1_scores     = cross_val_score(lsvc_l1, X, Y, cv=cv1, n_jobs=1)
print(lsvc_l1_scores)
print(np.mean(lsvc_l1_scores))
# Do NOT run again
lsvc_l2_scores     = cross_val_score(lsvc_l2, X, Y, cv=cv1, n_jobs=1)
print(lsvc_l2_scores)
print(np.mean(lsvc_l2_scores))
# Do NOT run again
lsvc_l3_scores     = cross_val_score(lsvc_l3, X, Y, cv=cv1, n_jobs=1)
print(lsvc_l3_scores)
print(np.mean(lsvc_l3_scores))
# Do NOT run again
lsvc_l4_scores     = cross_val_score(lsvc_l4, X, Y, cv=cv1, n_jobs=1)
print(lsvc_l4_scores)
print(np.mean(lsvc_l4_scores))
# Do NOT run again
lsvc_l5_scores     = cross_val_score(lsvc_l5, X, Y, cv=cv1, n_jobs=1)
print(lsvc_l5_scores)
print(np.mean(lsvc_l5_scores))
# Do NOT run again
lsvc_l6_scores     = cross_val_score(lsvc_l6, X, Y, cv=cv1, n_jobs=1)
print(lsvc_l6_scores)
print(np.mean(lsvc_l6_scores))<jupyter_output><empty_output><jupyter_text>finalized classifiers Applying finalized classifier on the actual test data and save them<jupyter_code>class arguments():
    def __init__(self):
        self.DATAPATH = '../data/'

args = arguments()

train = pd.read_csv(args.DATAPATH + 'studentspen-train.csv')
test = pd.read_csv(args.DATAPATH + 'studentsdigits-test.csv')

X_train = train[['x3', 'y3', 'x4', 'y4', 'x5', 'y5', 'x6', 'y6']].values
y_train = train['Digit'].values

X_test = test[['x3', 'y3', 'x4', 'y4', 'x5', 'y5', 'x6', 'y6']].values


print(X_train.shape)
print(y_train.shape)
print(X_test.shape)
#---------------------------------------------------------------------------------------------------
# Linear SVC
#---------------------------------------------------------------------------------------------------
svm_final_1 = make_pipeline(StandardScaler(), LinearSVC(loss='squared_hinge', random_state=rand_state,\
                                                        C=8, max_iter=max_iter, class_weight='balanced'))

svm_final_1_scores  = cross_val_score(svm_final_1, X, Y, cv=cv1, n_jobs=1)
print('one vs rest')
print(svm_final_1_scores)
print(np.mean(svm_final_1_scores))


#---------------------------------------------------------------------------------------------------
# SVC with one vs rest wrapper
#---------------------------------------------------------------------------------------------------

svm_final_2 = make_pipeline(StandardScaler(), SVC(kernel='linear', random_state=rand_state, \
                                                  C=8, max_iter=max_iter, class_weight='balanced'))
svm_final_2_scores  = cross_val_score(svm_final_2, X, Y, cv=cv1, n_jobs=1)
print('one vs one')
print(svm_final_2_scores)
print(np.mean(svm_final_2_scores))

#----------------------------------
svm_final_3 = OneVsRestClassifier(svm_final_2)
svm_final_3_scores     = cross_val_score(svm_final_3, X, Y, cv=cv1, n_jobs=1)
print('one vs rest')
print(svm_final_3_scores)
print(np.mean(svm_final_3_scores))
#---------------------------------------------------------------------------------------------------
# One vs Rest
#---------------------------------------------------------------------------------------------------
svm_final_1.fit(X_train, y_train)
y_pred = svm_final_1.predict(X_test)


SAVE_RESULT_ROOT = '../results/'
fn_str = SAVE_RESULT_ROOT + 'MaryZolfaghar_preds_Q3_OneVsRest.txt'
a_file = open(fn_str, "w")

for row in y_pred:
    res = str(row)
    a_file.write(str(int(float(res)))+"\n")
a_file.close()
print('done')

#---------------------------------------------------------------------------------------------------
# One Vs One
#---------------------------------------------------------------------------------------------------
svm_final_2.fit(X_train, y_train)
y_pred_ovo = svm_final_2.predict(X_test)


SAVE_RESULT_ROOT = '../results/'
fn_str = SAVE_RESULT_ROOT + 'MaryZolfaghar_preds_Q3_OneVsOne.txt'
a_file = open(fn_str, "w")

for row in y_pred_ovo:
    res = str(row)
    a_file.write(str(int(float(res)))+"\n")
a_file.close()
print('done')<jupyter_output>done
done<jupyter_text>Tuning parameters for one vs rest classifier<jupyter_code>class arguments():
    def __init__(self):
        self.DATAPATH = '../data/'

args = arguments()

train = pd.read_csv(args.DATAPATH + 'studentspen-train.csv')
test = pd.read_csv(args.DATAPATH + 'studentsdigits-test.csv')

X = train[['x3', 'y3', 'x4', 'y4', 'x5', 'y5', 'x6', 'y6']].values
Y = train['Digit'].values
print(X.shape)
print(Y.shape)


# n_split for linear = 10 and for polynomial = 6
n_split_linear=10
n_split_poly=6

cv1 = StratifiedShuffleSplit(n_splits=n_split_linear, random_state=rand_state) 
cv2 = StratifiedShuffleSplit(n_splits=n_split_poly, random_state=rand_state) 





bsvc = SVC(kernel='linear')
lsvc = LinearSVC()
ovrsvc = OneVsRestClassifier(bsvc)

ns_ranegs = [3, 6, 9, 12]

for ns in ns_ranegs:
    cv3 = StratifiedShuffleSplit(n_splits=ns, random_state=rand_state) 
    ovrsvc_scores    = cross_val_score(ovrsvc, X, Y, cv=cv3, n_jobs=1)
    print('for num of splits:', ns)
    print(ovrsvc_scores)
    print(np.mean(ovrsvc_scores))
    print('\n')



# (3747, 8)
# (3747,)
# for num of splits: 3
# [0.744      0.752      0.74666667]
# 0.7475555555555555


# for num of splits: 6
# [0.744      0.752      0.74666667 0.70666667 0.70933333 0.75466667]
# 0.7355555555555555


# for num of splits: 9
# [0.744      0.752      0.74666667 0.70666667 0.70933333 0.75466667
#  0.744      0.72266667 0.72266667]
# 0.7336296296296297


# for num of splits: 12
# [0.744      0.752      0.74666667 0.70666667 0.70933333 0.75466667
#  0.744      0.72266667 0.72266667 0.75733333 0.728      0.71466667]
# 0.7335555555555556
class arguments():
    def __init__(self):
        self.DATAPATH = '../data/'

args = arguments()

train = pd.read_csv(args.DATAPATH + 'studentspen-train.csv')
test = pd.read_csv(args.DATAPATH + 'studentsdigits-test.csv')

X = train[['x3', 'y3', 'x4', 'y4', 'x5', 'y5', 'x6', 'y6']].values
Y = train['Digit'].values
print(X.shape)
print(Y.shape)


# n_split for linear = 10 and for polynomial = 6
n_split_linear=10
n_split_poly=6

cv1 = StratifiedShuffleSplit(n_splits=n_split_linear, random_state=rand_state) 
cv2 = StratifiedShuffleSplit(n_splits=n_split_poly, random_state=rand_state) 





bsvc = SVC(kernel='linear')
lsvc = LinearSVC()
ovrsvc = OneVsRestClassifier(bsvc)

ns_ranges = [3, 6]
ts_ranges = [0.1, 0.2, 0.3, 0.4]
for ns in ns_ranges:
    for ts in ts_ranges:
        cv3 = StratifiedShuffleSplit(n_splits=ns, random_state=rand_state, test_size=ts) 
        ovrsvc_scores    = cross_val_score(ovrsvc, X, Y, cv=cv3, n_jobs=1)
        print('for num of splits and test size:', ns, ts)
        print(ovrsvc_scores)
        print(np.mean(ovrsvc_scores))
        print('\n')




# (3747, 8)
# (3747,)
# for num of splits and test size: 3 0.1
# [0.744      0.752      0.74666667]
# 0.7475555555555555


# for num of splits and test size: 3 0.2
# [0.744      0.73466667 0.73466667]
# 0.7377777777777778


# for num of splits and test size: 3 0.3
# [0.74933333 0.75377778 0.73866667]
# 0.7472592592592592


# for num of splits and test size: 3 0.4
# [0.73849233 0.73715811 0.73782522]
# 0.7378252168112075


# for num of splits and test size: 6 0.1
# [0.744      0.752      0.74666667 0.70666667 0.70933333 0.75466667]
# 0.7355555555555555


# for num of splits and test size: 6 0.2
# [0.744      0.73466667 0.73466667 0.72133333 0.728      0.72933333]
# 0.7319999999999999


# for num of splits and test size: 6 0.3
# [0.74933333 0.75377778 0.73866667 0.74044444 0.71288889 0.72177778]
# 0.7361481481481481


# for num of splits and test size: 6 0.4
# [0.73849233 0.73715811 0.73782522 0.73248833 0.72981988 0.73849233]
# 0.7357126973537914<jupyter_output><empty_output><jupyter_text>Tuning parameter C for SVC for  kernel = 'linear'C = 8<jupyter_code># Do NOT run again

scores=[]
range_params = np.array([2,4,6,8,9,10])
for ic, cc in enumerate(range_params):
    lsvc3 = make_pipeline(StandardScaler(), LinearModel(SVC(kernel='linear', C=cc, \
                                                            random_state=rand_state, \
                                                            max_iter=max_iter)))           

    lsvc3_scores     = cross_val_score(lsvc3, X, Y, cv=cv1)
    sc = np.mean(lsvc3_scores)
    scores.append(sc)
    print('C is ', cc, 'gives score', sc)
    

# Do NOT run again

bsvc = SVC(kernel='linear', C=8, random_state=rand_state, max_iter=max_iter)
bsvc_scores     = cross_val_score(bsvc, X, Y, cv=cv1, n_jobs=1)
print(bsvc_scores)
print(np.mean(bsvc_scores))<jupyter_output><empty_output><jupyter_text>Best model so far<jupyter_code># Do NOT run again

# checking class_weight='balanced' -> does not matter 
bsvc = SVC(kernel='linear', C=8, random_state=rand_state, max_iter=max_iter,\
           class_weight='balanced')
bsvc_scores     = cross_val_score(bsvc, X, Y, cv=cv1, n_jobs=1)
print(bsvc_scores)
print(np.mean(bsvc_scores))
# Do NOT run again

# checking gamma -> does not matter to set to a specific number
range_params =np.logspace(-1, 10, 10)
for al in range_params:
    bsvc = SVC(kernel='linear', C=8, random_state=rand_state, max_iter=max_iter,\
               class_weight='balanced', gamma=al)
    bsvc_scores     = cross_val_score(lsvc3, X, Y, cv=cv1, n_jobs=1)
    print(np.mean(bsvc_scores))
# Do NOT run again

# checking gamma -> does not matter to be scale or auto
bsvc = SVC(kernel='linear', C=8, random_state=rand_state, max_iter=max_iter,\
           class_weight='balanced', gamma='auto')
bsvc_scores     = cross_val_score(lsvc3, X, Y, cv=cv1, n_jobs=1)
print(np.mean(bsvc_scores))

# checking gamma
bsvc = SVC(kernel='linear', C=8, random_state=rand_state, max_iter=max_iter,\
           class_weight='balanced', gamma='scale')
bsvc_scores     = cross_val_score(lsvc3, X, Y, cv=cv1, n_jobs=1)
print(np.mean(bsvc_scores))
# Do NOT run again

# check deafult for cross validation, setting cv to integer 10, it will be kfold or kstratified 

bsvc = SVC(kernel='linear', C=8, random_state=rand_state, max_iter=max_iter)
bsvc_scores     = cross_val_score(lsvc3, X, Y, cv=10)
print(np.mean(bsvc_scores))<jupyter_output>0.8551056860183566<jupyter_text>Checking other classifiersSVC is way better than linearSVC<jupyter_code>lsvc_scores     = cross_val_score(lsvc, X, Y, cv=cv1, n_jobs=1)
print(lsvc_scores)
# lsvc_scores     = cross_val_score(lsvc, X, Y, cv=cv1, n_jobs=1)
print(lsvc_scores)
lsvc_l1_scores  = cross_val_score(lsvc_l1, X, Y, cv=cv1, n_jobs=1)
print(lsvc_l1_scores)<jupyter_output>[0.736      0.752      0.73333333 0.71466667 0.712      0.74133333
 0.73866667 0.712      0.73866667 0.73333333]<jupyter_text>Conclusions1. It might be easier to use GridSearch, but faster to do it separately to have a general idea1.1. So far I got C=8 for kernel ='linear', and n_splits=10 gave me the best result, gamma was not affecting the results1.2. I tried to double check that with 2. It might be a better cross validation to do a nested and non-nested cross validation3. For being able to plot I might need to use cross predict: https://scikit-learn.org/stable/auto_examples/model_selection/plot_cv_predict.htmlsphx-glr-auto-examples-model-selection-plot-cv-predict-py4. SVC was way better than linearSVC5. For transfer learning i might be able to use cross predict  Question 3Question 3) Submit your predictions on this test set, one prediction per line in the order given studentsdigits-test.csvPreview the document (10 points)  Checking classifiers after adding kernelseems kernel='poly' is the best one<jupyter_code>def plot_multiclass_hyperplane(X_plt, y_plt, clf_src, use_annotate):
    #----------------------------------------------------------------------------------               
    fig= plt.figure(figsize=(13,13))
    ax = fig.add_subplot(111)
    #----------------------------------------------------------------------------------               
    pca = PCA(n_components=2).fit(X_plt)
    pca_2d = pca.transform(X_plt)
    #----------------------------------------------------------------------------------               
    for i in range(0, pca_2d.shape[0]):
        if y_plt[i] == 0:
            c0 = plt.scatter(pca_2d[i,0],pca_2d[i,1],c='r',  marker='+')
        elif y_plt[i] == 1:
            c1 = plt.scatter(pca_2d[i,0],pca_2d[i,1],c='g',    marker='o')
        elif y_plt[i] == 2:
            c2 = plt.scatter(pca_2d[i,0],pca_2d[i,1],c='b',    marker='s')
        elif y_plt[i] == 3:
            c3 = plt.scatter(pca_2d[i,0],pca_2d[i,1],c='y',    marker='p')
        elif y_plt[i] == 4:
            c4 = plt.scatter(pca_2d[i,0],pca_2d[i,1],c='c',    marker='x')
        elif y_plt[i] == 5:
            c5 = plt.scatter(pca_2d[i,0],pca_2d[i,1],c='teal',    marker='P')
        elif y_plt[i] == 6:
            c6 = plt.scatter(pca_2d[i,0],pca_2d[i,1],c='peru',  marker='D')
        elif y_plt[i] == 7:
            c7 = plt.scatter(pca_2d[i,0],pca_2d[i,1],c='lime',    marker='X')
        elif y_plt[i] == 8:
            c8 = plt.scatter(pca_2d[i,0],pca_2d[i,1],c='fuchsia',    marker='>')
        elif y_plt[i] == 9:
            c9 = plt.scatter(pca_2d[i,0],pca_2d[i,1],c='seagreen',    marker='<')
        if use_annotate:
            ax.annotate(str(y_plt[i]), (pca_2d[i,0],pca_2d[i,1]))
    #----------------------------------------------------------------------------------               
    plt.legend([c0, c1, c2, c3, c4, c5, c6, c7, c8, c9],\
              ['digit 0', 'digit 1', 'digit 2','digit 3', 'digit 4', 'digit 5',\
               'digit 6', 'digit 7', 'digit 8','digit 9' ])
    #----------------------------------------------------------------------------------               
    svclassifier_2d = clf_src
    svclassifier_2d.fit(pca_2d, y_plt)
    #----------------------------------------------------------------------------------               
    x_min, x_max = pca_2d[:, 0].min() - 1,   pca_2d[:,0].max() + 1
    y_min, y_max = pca_2d[:, 1].min() - 1,   pca_2d[:,1].max() + 1
    xx, yy = np.meshgrid(np.arange(x_min, x_max, 10),   np.arange(y_min, y_max, 10))

    Z = svclassifier_2d.predict(np.c_[xx.ravel(),  yy.ravel()])
    Z = Z.reshape(xx.shape)
    plt.contour(xx, yy, Z)
    plt.title('Support Vector Machine Decision Surface', fontweight='bold', fontsize=22)
#     plt.axis('off')
    plt.show()
    #----------------------------------------------------------------------------------               
    print('done')
class arguments():
    def __init__(self):
        self.DATAPATH = '../data/'

args = arguments()

train = pd.read_csv(args.DATAPATH + 'studentspen-train.csv')
test = pd.read_csv(args.DATAPATH + 'studentsdigits-test.csv')

X = train[['x3', 'y3', 'x4', 'y4', 'x5', 'y5', 'x6', 'y6']].values
Y = train['Digit'].values
print(X.shape)
print(Y.shape)
rand_state=45
max_iter=100000
# n_split for linear = 10 and for polynomial = 6
n_split_linear=10
n_split_poly=6

cv1 = StratifiedShuffleSplit(n_splits=n_split_linear, random_state=rand_state) 
cv2 = StratifiedShuffleSplit(n_splits=n_split_poly, random_state=rand_state) 
cv3 = StratifiedShuffleSplit(n_splits=3, random_state=rand_state) 
psvc = SVC(kernel='poly', gamma='scale')
psvc2 = SVC(kernel='poly', gamma='auto')
rsvc = SVC(kernel='rbf', gamma='auto')
ssvc = SVC(kernel='sigmoid', gamma='auto')
ovrsvc = OneVsRestClassifier(psvc)
psvc_scores     = cross_val_score(psvc, X, Y, cv=cv2, n_jobs=1)
print(psvc_scores)
print(np.mean(psvc_scores))
psvc2_scores    = cross_val_score(psvc2, X, Y, cv=cv2, n_jobs=1)
print(psvc2_scores)
print(np.mean(psvc2_scores))
rsvc_scores    = cross_val_score(rsvc, X, Y, cv=cv2, n_jobs=1)
print(rsvc_scores)
print(np.mean(rsvc_scores))
ssvc_scores    = cross_val_score(ssvc, X, Y, cv=cv2, n_jobs=1)
print(ssvc_scores)
print(np.mean(ssvc_scores))
ovrsvc_scores    = cross_val_score(ovrsvc, X, Y, cv=cv3, n_jobs=1)
print(ovrsvc_scores)
print(np.mean(ovrsvc_scores))
plot_multiclass_hyperplane(X_plt=X.copy(), y_plt=Y.copy(), clf_src=ovrsvc, use_annotate=False)
for trg_train_index, trg_test_index in cv1.split(X17, Y17):<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.19563266527071493, 'fraction_numerical': 0.0657094426164124, 'mean_word_length': 3.048234106962664, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 34, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 10, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '159359', 'license': 'Apache-2.0', 'n_tokens_mistral': 8460, 'n_tokens_neox': 7542, 'n_words': 1479, 'path': 'jupyters/tinkerings/tinkering_SVM_Question1_2_3.ipynb', 'repo_name': 'MaryZolfaghar/HandWrittenDigitPrediction', 'size': '20058'}