<jupyter_start><jupyter_text>Actual Data Classification  Dealing with Data Crap  Load Data<jupyter_code># with open("/Users/collopa/Desktop/nonlinear/project/Colin_Eli_Kai_Math178_Final/data/train_vectors/train_features.pkl", "rb") \
#         as input_file:  train_features = pickle.load(input_file)
    
# with open("/Users/collopa/Desktop/nonlinear/project/Colin_Eli_Kai_Math178_Final/data/train_vectors/train_labels.pkl", "rb") \
#     as input_file:  train_labels = pickle.load(input_file)

# with open("/Users/collopa/Desktop/nonlinear/project/Colin_Eli_Kai_Math178_Final/data/test_vectors/test_features.pkl", "rb") \
#         as input_file:  test_features = pickle.load(input_file)
    
# with open("/Users/collopa/Desktop/nonlinear/project/Colin_Eli_Kai_Math178_Final/data/test_vectors/test_labels.pkl", "rb") \
#     as input_file:  test_labels = pickle.load(input_file)

test_path = '/Volumes/GoogleDrive/My Drive/<NAME>/Work/Summer 2020/project_data/test_vectors/test_featureMat.csv'
train_path = '/Volumes/GoogleDrive/My Drive/<NAME>/Work/Summer 2020/project_data/train_vectors/train_featureMat.csv'

#load the matrix
trainMat = pd.read_csv(train_path)
testMat = pd.read_csv(test_path)<jupyter_output><empty_output><jupyter_text>Select Features and Extract Labels<jupyter_code>#select rows to use
trainMat_ = trainMat.copy()
testMat_ = testMat.copy()
#trainMat_ = trainMat_[np.logical_or(trainMat.label == 'wlk',trainMat.label == 'sit')]
#testMat_ = testMat_[np.logical_or(testMat.label == 'wlk',testMat.label == 'sit')]

#extract labels
train_labels = trainMat_['label'].values
del trainMat_['label']

test_labels = testMat_['label'].values
del testMat_['label']


#extract appropriate features
all_feats = trainMat_.columns
acc_feats = [f for f in all_feats if 'a_' in f]
train_features = trainMat_[acc_feats].to_numpy()
test_features = testMat_[acc_feats].to_numpy()<jupyter_output><empty_output><jupyter_text>SVM<jupyter_code>t = time.time()

clf = svm.SVC(decision_function_shape='ovo')
clf.fit(train_features, train_labels)

elapsed = time.time() - t
print(elapsed)
t = time.time()

pred = clf.predict(test_features)
elapsed = time.time() - t

print(elapsed)<jupyter_output>26.17463183403015<jupyter_text>Kai thinks this will take 30 minutes. I (Colin) predict 10. Update: Colin won.<jupyter_code>print("Accuracy:", metrics.accuracy_score(test_labels, pred))
unique_elements, counts_elements = np.unique(pred, return_counts=True)
print("Frequency of unique values of the predictions:")
print(np.asarray((unique_elements, counts_elements)))
unique_elements, counts_elements = np.unique(test_labels, return_counts=True)
print("Frequency of unique values of the labels:")
print(np.asarray((unique_elements, counts_elements)))<jupyter_output>Frequency of unique values of the predictions:
[['dws' 'jog' 'sit' 'std' 'ups' 'wlk']
 [292 472 2487 340 378 986]]
Frequency of unique values of the labels:
[['dws' 'jog' 'sit' 'std' 'ups' 'wlk']
 [315 429 1589 1238 400 984]]<jupyter_text>knn<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
neigh = KNeighborsClassifier(n_neighbors=3)
neigh.fit(train_features, train_labels)

pred = neigh.predict(test_features)
unique_elements, counts_elements = np.unique(pred, return_counts=True)
print("Frequency of unique values of the predictions:")
print(np.asarray((unique_elements, counts_elements)))
unique_elements, counts_elements = np.unique(test_labels, return_counts=True)
print("Frequency of unique values of the labels:")
print(np.asarray((unique_elements, counts_elements)))
print("Accuracy:", metrics.accuracy_score(test_labels, pred))<jupyter_output>Accuracy: 0.856710393541877<jupyter_text>logistic regression<jupyter_code>from sklearn.linear_model import LogisticRegression

clf = LogisticRegression().fit(train_features, train_labels)
t = time.time()

pred = clf.predict(test_features)
elapsed = time.time() - t

print(elapsed)
print("Accuracy:", metrics.accuracy_score(test_labels, pred))

unique_elements, counts_elements = np.unique(pred, return_counts=True)
print("Frequency of unique values of the predictions:")
print(np.asarray((unique_elements, counts_elements)))
unique_elements, counts_elements = np.unique(test_labels, return_counts=True)
print("Frequency of unique values of the labels:")
print(np.asarray((unique_elements, counts_elements)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12353884941553976, 'fraction_numerical': 0.022003208801283522, 'mean_word_length': 7.540117416829745, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '340368', 'license': 'Unlicense', 'n_tokens_mistral': 1622, 'n_tokens_neox': 1499, 'n_words': 278, 'path': 'prototyping/skikit_supervised.ipynb', 'repo_name': 'eliweissler/Colin_Eli_Kai_Math_178_Final', 'size': '4373'}