<jupyter_start><jupyter_text>Support Vector Machine: Predicting community membership of Perdiz arrow points<jupyter_code># load analysis packages
import pandas as pd
%matplotlib inline
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import cross_val_score
from sklearn import svm
from sklearn.svm import SVC
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from sklearn.metrics import roc_auc_score, accuracy_score
from sklearn import metrics

# read data
data = pd.read_csv('perdizsite.csv')
data.head()<jupyter_output><empty_output><jupyter_text>select features and response<jupyter_code># attributes for analysis
feature_cols = ['maxl', 'maxw', 'maxth', 'maxstl', 'maxstw']
X = data[feature_cols]

# cast from string to int
reg_num = {'north':0, 'south':1}
data['reg_num'] = data.region.map(reg_num)
data.head()
y = data.reg_num<jupyter_output><empty_output><jupyter_text>ensure that features and responses are numeric<jupyter_code>X.dtypes
y.dtypes<jupyter_output><empty_output><jupyter_text>split data for train/test<jupyter_code># split data into train/test sets (75/25 split)
X_train, X_test, y_train, y_test = train_test_split(X, y,
                                                    test_size = 0.25,
                                                    random_state = 0)

print('X_train: ', X_train.shape)
print('X_test: ', X_test.shape)
print('y_train:', y_train.shape)
print('y_test: ', y_test.shape)<jupyter_output>X_train:  (50, 5)
X_test:  (17, 5)
y_train: (50,)
y_test:  (17,)<jupyter_text>decrease sensitivity of algorithm to outliers through standardizing features<jupyter_code>stdsc = StandardScaler()
X_train_std = stdsc.fit_transform(X_train)
X_test_std = stdsc.transform(X_test)<jupyter_output><empty_output><jupyter_text>create SVM classifier with linear kernel<jupyter_code>clf = svm.SVC(kernel = 'linear')
clf.fit(X_train_std, y_train)<jupyter_output><empty_output><jupyter_text>grid search and nested cross validation of training dataset<jupyter_code># grid search
pipe_svc = Pipeline([('scl', StandardScaler()), ('clf', SVC(random_state = 0))])
param_range = [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]
param_grid = [{'clf__C': param_range,
              'clf__kernel': ['linear']},
             {'clf__C': param_range,
             'clf__gamma': param_range,
             'clf__kernel': ['rbf']}]
gs = GridSearchCV(estimator = pipe_svc,
                  param_grid = param_grid,
                  scoring = 'accuracy',
                  cv = 10,
                  n_jobs = 1)
gs = gs.fit(X_train_std, y_train)
print('Grid Search Best Score: ', gs.best_score_)
print('Grid Search Best Parameters: ', gs.best_params_)
# use the test dataset to estimate model performance
clf = gs.best_estimator_
clf.fit(X_train_std, y_train)
clf.score(X_test_std, y_test)

# nested cross validation
gs = GridSearchCV(estimator = pipe_svc,
                 param_grid = param_grid,
                 scoring = 'accuracy',
                 cv = 10,
                 n_jobs = 1)
scores = cross_val_score(gs, X_train_std, y_train,
                         scoring = 'accuracy',
                         cv = 10)
print('Cross Validation Scores: ', scores)
print('Cross Validation Mean Score: ', scores.mean())<jupyter_output>Grid Search Best Score:  0.8600000000000001
Grid Search Best Parameters:  {'clf__C': 10.0, 'clf__kernel': 'linear'}
Cross Validation Scores:  [0.8 1.  0.4 1.  0.8 0.8 1.  1.  0.8 1. ]
Cross Validation Mean Score:  0.8600000000000001<jupyter_text>make predictions + evaluate accuracy<jupyter_code>y_pred = clf.predict(X_test_std)
print('Receiver Operator Curve Score: ', roc_auc_score(y_true = y_test,
                                                       y_score = y_pred))
print('Accuracy Score: ', accuracy_score(y_test, y_pred))
print('Precision: ', metrics.precision_score(y_test, y_pred))
print('Recall: ', metrics.recall_score(y_test, y_pred))
# plot ROC curve
fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred)
plt.plot(fpr, tpr)
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.0])
plt.title('ROC curve for Perdiz classifier')
plt.xlabel('False Positive Rate (1 - Specificity)')
plt.ylabel('True Positive Rate (Sensitivity)')
plt.grid(True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11497938616582684, 'fraction_numerical': 0.02702702702702703, 'mean_word_length': 3.8957399103139014, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '265213', 'license': 'MIT', 'n_tokens_mistral': 1598, 'n_tokens_neox': 1476, 'n_words': 344, 'path': 'ml/perdiz-svm.ipynb', 'repo_name': 'seldenlab/perdiz3', 'size': '4366'}