<jupyter_start><jupyter_text>reshaffling the data<jupyter_code>np.random.seed(42)
train = df.sample(frac=1)
Y = train['Swarm_Behaviour']
X = train.drop(['Swarm_Behaviour'],axis=1)
print(X.shape, Y.shape)<jupyter_output>(23309, 2400) (23309,)<jupyter_text>Standardize data after training and testing split<jupyter_code># standardize the test dataset using mean and std from x_train to aviod leaking
x_train, x_test, y_train, y_test = train_test_split(X, Y, train_size=0.8, test_size = 0.2)
mean = x_train.mean()
std = x_train.std()
x_train = (x_train - mean) / std
x_test = (x_test - mean) / std
fig, axes = plt.subplots(nrows=1, ncols=2, sharey=False, figsize=(12, 4))
fig.suptitle('distribution of labels in swarm behavior dataset', fontsize=16)
axes[0].hist(y_train)
axes[0].title.set_text('in training dataset')
axes[0].set_xlabel("Binary lables")
axes[0].set_ylabel("Number of Samples")
axes[1].set_xlabel("Binary lables")
axes[1].set_ylabel("Number of Samples")
axes[1].hist(y_test)
axes[1].title.set_text('in testing dataset')
save_fig("dist_sw")<jupyter_output>Saving figure dist_sw<jupyter_text>Choosing the number of dimensions to reduce<jupyter_code>pca = PCA()
pca.fit(x_train)
#X_reduced = pca.fit_transform(X)
cumsum = np.cumsum(pca.explained_variance_ratio_)
d = np.argmax(cumsum >= 0.95) + 1
d
#X_reduced.shape
exp_var_pca = pca.explained_variance_ratio_
exp_var_pca
cumsum.shape
plt.figure(figsize=(10,4))
plt.bar(range(0,len(exp_var_pca)), exp_var_pca, alpha=0.5, align='center', label='Individual explained variance')
plt.step(range(0,len(cumsum)), cumsum, where='mid',label='Cumulative explained variance')
plt.axis([0, 750, 0, 1])
for i, line in enumerate(exp_var_pca):
    x, y = i, line
    text = ' {:.2f}'.format(y)
    plt.annotate(text, xy=(x+0.05, y-0.05))
plt.xlabel("Dimensions")
#plt.legend(loc='upper right')
plt.ylabel("Explained Variance Ratio")
plt.plot([d, d], [0, 0.95], "k:")
plt.plot([0, d], [0.95, 0.95], "k:")

plt.plot(d, 0.95, "ko")
plt.annotate("95% variance captured", xy=(d, 0.90), xytext=(d, 0.7),
             arrowprops=dict(arrowstyle="->"), fontsize=12)
plt.grid(True)
plt.title("Swarm behavior--changes of explained variances by dimensions")

plt.tight_layout()
save_fig("sw_pca_captured")<jupyter_output>Saving figure sw_pca_captured<jupyter_text>use PCA to visualize high-dimensional dataset<jupyter_code>pca = PCA(n_components=3)
df_2D = pca.fit_transform(x_train)

df_2D.shape
Y.shape
pca.explained_variance_ratio_
df_2D = pd.DataFrame(df_2D)
df_2D['label'] = pd.DataFrame.reset_index(pd.DataFrame(y_train),drop=True)
df_2D = df_2D.rename(columns={0:'PC1',1:'PC2',2:'PC3'})
df_2D.head
df_2D['label'].unique()
g = sns.scatterplot(x=df_2D['PC1'], y=df_2D['PC2'], hue=df_2D['label'], palette=['r','b'])
legend_labels, _= g.get_legend_handles_labels()
g.legend(legend_labels,['not aligned', 'aligned'])
g.set(title='1st and 2st PCA of swarm behavior training dataset')
save_fig("sw_pca_2d")

#ax.set_xlabel("PC1")
#ax.set_ylabel("PC2")
#ax.legend(loc='upper right')
#ax.set_title("1st and 2st PCA of swarm behavior training dataset")
from mpl_toolkits.mplot3d import Axes3D
from matplotlib.colors import ListedColormap

color_dict={0:'r', 1:'b'}
fig = plt.figure(figsize=(6, 5))

ax = plt.subplot(111, projection='3d')

ax.scatter(df_2D['PC1'], df_2D['PC2'], df_2D['PC3'], c = [color_dict[i] for i in df_2D['label']], \
           label=list(i for i in pd.unique(df_2D['label'])))
legend_labels, _= ax.get_legend_handles_labels()
print(_)
ax.set_xlabel('PC1')
ax.set_ylabel('PC2')
ax.set_zlabel('PC3')
ax.set_title("1st, 2st, and 3rd PCA of swarm behavior training dataset")
#ax.legend()
save_fig("sw_pca_3d")<jupyter_output>['[0, 1]']
Saving figure sw_pca_3d<jupyter_text>Start to train a binary classifier<jupyter_code>t0 = time.time()
clf = RandomForestClassifier(max_depth=3, random_state=42)
clf.fit(x_train,y_train)
y_pred=clf.predict(x_test)
t1 = time.time()
time_before_pca = t1-t0
print("Random forest took {:.5f}s.".format(t1 - t0))
confusion_matrix(y_test, y_pred)
f1_before_pca = f1_score(y_test, y_pred)
f1_before_pca
pca = PCA(n_components = d)
X_reduced = pca.fit_transform(x_train)
X_test_reduced = pca.fit_transform(x_test)
X_recovered = pca.inverse_transform(X_reduced)
X_recovered
from sklearn.metrics.pairwise import cosine_similarity,cosine_distances
result=cosine_similarity(np.asarray(X_recovered).reshape(1,-1),np.asarray(x_train).reshape(1,-1))
result
#1 - sp.distance.cosine(X_recovered, x_train)
t0 = time.time()
clf2 = RandomForestClassifier(max_depth=3, random_state=42)
clf2.fit(X_reduced,y_train)
y_pred2=clf2.predict(X_test_reduced)
t1 = time.time()
time_after_pca = t1-t0

print("Random forest took {:.5f}s.".format(t1 - t0))
confusion_matrix(y_test, y_pred2)
f1_after_pca = f1_score(y_test, y_pred2)
f1_after_pca
f1_variance = (f1_after_pca-f1_before_pca)/f1_before_pca*100
time_variance = (time_after_pca-time_before_pca)/time_before_pca*100

print(f1_variance, time_variance)
from sklearn.metrics import roc_auc_score

auc_score1 = roc_auc_score(y_test, y_pred)
auc_score2 = roc_auc_score(y_test, y_pred2)

auc_score2
n_classes = 2
fpr = dict()
tpr = dict()
roc_auc = dict()
fpr[0], tpr[0], _ = roc_curve(y_test, y_pred)
roc_auc[0] = auc(fpr[0], tpr[0])
fpr[1], tpr[1], _ = roc_curve(y_test, y_pred2)
roc_auc[1] = auc(fpr[1], tpr[1])

#fpr["micro1"], tpr["micro1"], _ = roc_curve(y_test.ravel(), y_pred.ravel())
#roc_auc["micro1"] = auc(fpr["micro1"], tpr["micro1"])
#all_fpr = np.unique(np.concatenate([fpr[i] for i in range(n_classes)]))
#lw = 1
#mean_tpr = np.zeros_like(all_fpr)
#print(all_fpr)
plt.figure()
lw = 2
plt.plot(
    fpr[0],
    tpr[0],
    color="darkorange",
    lw=lw,
    label="ROC curve before PCA (area = %0.2f)" % roc_auc[0],
)
plt.plot(
    fpr[1],
    tpr[1],
    color="red",
    linestyle='-.',
    lw=lw,
    label="ROC curve after PCA (area = %0.2f)" % roc_auc[1],
)
plt.plot([0, 1], [0, 1], color="lightblue", lw=lw, linestyle="--")
plt.xlim([0.0, 1.0])
plt.ylim([0.0, 1.05])
plt.xlabel("False Positive Rate")
plt.ylabel("True Positive Rate")
plt.title("ROC for swarm behavior datset")
plt.legend(loc="lower right")

save_fig("sw_roc")<jupyter_output>Saving figure sw_roc
{'fraction_non_alphanumeric': 0.16739376312813056, 'fraction_numerical': 0.03780901599612215, 'mean_word_length': 6.747183979974968, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '529670', 'license': 'MIT', 'n_tokens_mistral': 2810, 'n_tokens_neox': 2640, 'n_words': 465, 'path': 'pca-swarm.ipynb', 'repo_name': 'ynuozhang/pca_term_paper', 'size': '6189'}