<jupyter_start><jupyter_text>Load Data<jupyter_code>X_train = np.load("xtrain_spec.npy")
X_train = X_train.reshape(X_train.shape[0], X_train.shape[1] * X_train.shape[2])
train = np.load("train.npy")
y_train = train[:,0]
X_labeled = X_train[y_train != -1]
y_labeled = y_train[y_train != -1]
X_unlabeled = X_train[y_train == -1]
y_unlabeled = y_train[y_train == -1]

X_test = np.load("test_spec.npy")
X_test = X_test.reshape(X_test.shape[0], X_test.shape[1] * X_test.shape[2])<jupyter_output><empty_output><jupyter_text>PCA<jupyter_code>pca = PCA(n_components=1000)
pca.fit(X_train)
print(pca.explained_variance_ratio_)
cum = [pca.explained_variance_ratio_[0]]
for i in range(1, len(pca.explained_variance_ratio_)):
    cum.append(cum[-1] + pca.explained_variance_ratio_[i])
plt.title("Cumulative Proportion of Variance Explained")
plt.xlabel("Number of Principal Components")
plt.plot(cum)<jupyter_output><empty_output><jupyter_text>Random Forest Classification<jupyter_code>X_dimred = pca.transform(X_train)
X_labeled = X_dimred[y_train != -1]
y_labeled = y_train[y_train != -1]
X_unlabeled = X_dimred[y_train == -1]
y_unlabeled = y_train[y_train == -1]
means = [np.mean(X_dimred[y_train == k],axis=0) for k in range(10)]
for i in range(len(X_unlabeled)):
    y_unlabeled[i] = np.argmin([np.linalg.norm(X_unlabeled[i] - means[k]) for k in range(10)])
y_unlabeled[:10]
clf_xgb = XGBClassifier()
clf_xgb.fit(np.concatenate((X_labeled,X_unlabeled), axis=0), np.concatenate((y_labeled,y_unlabeled), axis=0))
predictions = clf_xgb.predict(pca.transform(X_test))
print("XGB_score: %f" % accuracy_score(clf_xgb.predict(X_labeled), y_labeled))<jupyter_output>XGB_score: 0.669701<jupyter_text>Write Predictions<jupyter_code>with open("predictions_xg.csv", "w") as f:
    f.write("Id,Prediction\n")
    for i, u in enumerate(predictions):
        f.write(str(i) + "," + str(int(u)) + "\n")
from sklearn.linear_model import LogisticRegression
clf_log = LogisticRegression()
clf_log.fit(np.concatenate((X_labeled,X_unlabeled), axis=0), np.concatenate((y_labeled,y_unlabeled), axis=0))
predictions_log = clf_log.predict(pca.transform(X_test))
print("LOG_score: %f" % accuracy_score(clf_log.predict(X_labeled), y_labeled))
clf_mlp = MLPClassifier()
clf_mlp.fit(np.concatenate((X_labeled,X_unlabeled), axis=0), np.concatenate((y_labeled,y_unlabeled), axis=0))
print("MLP score: %f" % accuracy_score(clf_mlp.predict(X_labeled), y_labeled))
clf_rf = RandomForestClassifier()
clf_rf.fit(np.concatenate((X_labeled,X_unlabeled), axis=0), np.concatenate((y_labeled,y_unlabeled), axis=0))
print("RF_score: %f" % accuracy_score(clf_rf.predict(X_labeled), y_labeled))
clf_rf_0 = RandomForestClassifier()
clf_rf_0.fit(X_labeled, y_labeled)
print("RF just labelled score: %f" % accuracy_score(clf_rf_0.predict(X_labeled), y_labeled))
X_labeled = X_train[y_train != -1]
y_labeled = y_train[y_train != -1]
X_unlabeled = X_train[y_train == -1]
y_unlabeled = y_train[y_train == -1]
means = [np.mean(X_train[y_train == k],axis=0) for k in range(10)]
for i in range(len(X_unlabeled)):
    y_unlabeled[i] = np.argmin([np.linalg.norm(X_unlabeled[i] - means[k]) for k in range(10)])
clf_rf_1 = RandomForestClassifier()
clf_rf_1.fit(np.concatenate((X_labeled,X_unlabeled), axis=0), np.concatenate((y_labeled,y_unlabeled), axis=0))
print("RF_1_score: %f" % accuracy_score(clf_rf.predict(X_labeled), y_labeled))
X_train = train[:,1:]
y_train = train[:,0]
pca_basic = PCA(n_components=1000)
pca_basic.fit(X_train)
X_train = pca_basic.transform(X_train)
X_labeled = X_train[y_train != -1]
y_labeled = y_train[y_train != -1]
X_unlabeled = X_train[y_train == -1]
y_unlabeled = y_train[y_train == -1]

means = [np.mean(X_train[y_train == k],axis=0) for k in range(10)]
for i in range(len(X_unlabeled)):
    y_unlabeled[i] = np.argmin([np.linalg.norm(X_unlabeled[i] - means[k]) for k in range(10)])
clf_rf_2 = RandomForestClassifier()
clf_rf_2.fit(np.concatenate((X_labeled,X_unlabeled), axis=0), np.concatenate((y_labeled,y_unlabeled), axis=0))
print("RF_2_score: %f" % accuracy_score(clf_rf.predict(pca_basic.transform(X_labeled)), y_labeled))<jupyter_output><empty_output><jupyter_text>Write Predictions<jupyter_code>with open("predictions_mlp.csv", "w") as f:
    f.write("Id,Prediction\n")
    for i, u in enumerate(predictions_mlp):
        f.write(str(i) + "," + str(int(u)) + "\n")<jupyter_output><empty_output><jupyter_text>Listening to the audioThis code enables you to listen to a sample sine wave audio.<jupyter_code>SAMPLE_RATE = 22050
train = np.load("train.npy")
train = train[:,1:]
y_train = train[:,0]
y_labeled_labels = y_train[y_train != -1] 
y_unlabled_labels
hist = [sum(predictions_log == i)/len(predictions_log) for i in range(10)]
real_hist = [sum(y_labeled == i)/len(y_labeled) for i in range(10)]
w=0.2
plt.bar(np.array(range(10))-[w] * 10, hist, w)
plt.bar(np.array(range(10)), hist_freq, w)
plt.bar(np.array(range(10))+[w] * 10, real_hist, w)
plt.title("Class Frequencies for Logistic Model")
plt.legend(['Model', 'Real', 'Reported'])
plt.xlabel('Class')
plt.ylabel('Frequency')
hist_mlp = [sum(predictions_mlp == i)/len(predictions_mlp) for i in range(10)]
real_hist = [sum(y_labeled == i)/len(y_labeled) for i in range(10)]
w=0.2
plt.bar(np.array(range(10))-[w] * 10, hist_mlp, w)
plt.bar(np.array(range(10)), hist_freq, w)
plt.bar(np.array(range(10))+[w] * 10, real_hist, w)
plt.title("Class Frequencies for Neural Net Model")
plt.legend(['Model', 'Real', 'Reported'])
plt.xlabel('Class')
plt.ylabel('Frequency')
np.array(hist_mlp) - np.array(real_hist)
hist_freq = np.array([13.61,2.77,13.23,9.22,10.98,13.12,0.22,10.96,12.24,13.65])/100<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1704686118479222, 'fraction_numerical': 0.028470380194518124, 'mean_word_length': 7.467065868263473, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '156507', 'license': 'MIT', 'n_tokens_mistral': 2533, 'n_tokens_neox': 2366, 'n_words': 365, 'path': 'P3/helpers.ipynb', 'repo_name': 'MikeBao99/cs181-practicals', 'size': '5655'}