<jupyter_start><jupyter_text>Data Prep<jupyter_code>df = pd.read_csv("../Phase1/main_data.csv", index_col=0)
df = df.dropna()
df
from sklearn.utils import shuffle
from sklearn.model_selection import train_test_split

X = df.drop('criticality', axis=1).values
Y = df['criticality'].values
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, stratify=Y, test_size=0.2, random_state=42)
# shuffled_data = shuffle(df.values, random_state=0)
# shuffled_data.shape

# test_index = int(shuffled_data.shape[0] * 0.8)

# X_train, X_test = shuffled_data[:test_index, :-1], shuffled_data[test_index:, :-1]
# Y_train, Y_test = shuffled_data[:test_index, -1], shuffled_data[test_index:, -1]
# shuffled_data.shape
print(X_train.shape, X_test.shape)<jupyter_output>(6568, 10) (1643, 10)<jupyter_text>Decision Tree<jupyter_code>from sklearn import tree

dt_clf = tree.DecisionTreeClassifier(random_state=0, max_depth=10)
dt_clf.fit(X_train, Y_train)

np.sum(dt_clf.predict(X_test) == Y_test) / len(X_test)<jupyter_output><empty_output><jupyter_text>Validation Curve<jupyter_code>#validation curve
from sklearn.model_selection import validation_curve

max_depth_list = range(5, 20)

train_scores, valid_scores = validation_curve(tree.DecisionTreeClassifier(),  X_train, Y_train, 
                                              param_name="max_depth",
                                              param_range=max_depth_list, 
                                              cv=5, 
                                              scoring = 'accuracy',
                                              verbose=1, n_jobs=-1
                                             )
print(train_scores.shape)

train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
valid_scores_mean = np.mean(valid_scores, axis=1)
valid_scores_std = np.std(valid_scores, axis=1)


xlabel = 'max depth'
ylabel = 'Accuracy'

plt_title = 'Validation Curve, DTClassifier'

fig = plt.figure()
ax = fig.add_subplot(111, xlabel=xlabel, ylabel=ylabel, title=plt_title)
plt.semilogx(max_depth_list, train_scores_mean, label="Training score",
             color="darkorange", lw=2)
plt.fill_between(max_depth_list, train_scores_mean - train_scores_std,
                 train_scores_mean + train_scores_std, alpha=0.2,
                 color="darkorange", lw=2)
plt.semilogx(max_depth_list, valid_scores_mean, label="Validation score",
             color="navy", lw=2)
plt.fill_between(max_depth_list, valid_scores_mean - valid_scores_std,
                 valid_scores_mean + valid_scores_std, alpha=0.2,
                 color="navy", lw=2)
plt.legend(loc="best")
plt.savefig('results/validation_curve_DT.jpg', bbox_inches='tight', dpi=200)
plt.show()<jupyter_output>[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.
[Parallel(n_jobs=-1)]: Done  75 out of  75 | elapsed:    1.8s finished<jupyter_text>Learning Curve<jupyter_code>#learning curve
from sklearn.model_selection import learning_curve

max_depth_list = range(5, 20)


ns_list, train_scores, validation_scores = learning_curve(
                                                   estimator = tree.DecisionTreeClassifier(), 
                                                   X = X_train, y = Y_train, 
                                                   train_sizes = max_depth_list, 
                                                   scoring = 'accuracy', 
                                                   n_jobs= -1
                                                   )

train_scores_mean =  train_scores.mean(axis = 1)
train_scores_std = np.std(train_scores, axis=1)
validation_scores_mean =  validation_scores.mean(axis = 1)
validation_scores_std  = validation_scores.std(axis = 1)

xlabel = 'Max Depth'
ylabel = 'Accuracy'

plt_title = 'Learning Curve, DecisionTreeClassifier'

fig = plt.figure()
ax = fig.add_subplot(111, xlabel=xlabel, ylabel=ylabel, title=plt_title)
ax.plot(max_depth_list, train_scores_mean, label = 'Training Score')
ax.plot(max_depth_list, validation_scores_mean, label = 'Validation Score')

plt.fill_between(max_depth_list, train_scores_mean - train_scores_std,
                 train_scores_mean + train_scores_std, alpha=0.2,
                 color="darkblue", lw=2)
plt.fill_between(max_depth_list, validation_scores_mean - validation_scores_std,
                 validation_scores_mean + validation_scores_std, alpha=0.2,
                 color="darkgreen", lw=2)


ax.legend(loc=0)
plt.savefig("results/learning_curve_DT.jpg", bbox_inches='tight', dpi=200)
plt.show()<jupyter_output><empty_output><jupyter_text>KNN (Neatest Neighbor Classification)**This takes a little bit of time**<jupyter_code>from sklearn import neighbors
from sklearn.pipeline import Pipeline

print(sorted(neighbors.KDTree.valid_metrics))
nca = neighbors.NeighborhoodComponentsAnalysis(random_state=42)
knn = neighbors.KNeighborsClassifier(n_neighbors=3)
nca_pipe = Pipeline([('nca', nca), ('knn', knn)])
nca_pipe.fit(X_train, Y_train)
nca_pipe.score(X_test, Y_test)
#validation curve
from sklearn.model_selection import validation_curve

n_neighbors_list = range(3, 15)

train_scores, valid_scores = validation_curve(neighbors.KNeighborsClassifier(metric='l1'),  X_train, Y_train, 
                                              param_name="n_neighbors",
                                              param_range=n_neighbors_list, 
                                              cv=5, 
                                              scoring = 'accuracy',
                                              verbose=1, n_jobs=-1
                                             )
print(train_scores.shape)

train_scores_mean = np.mean(train_scores, axis=1)
train_scores_std = np.std(train_scores, axis=1)
valid_scores_mean = np.mean(valid_scores, axis=1)
valid_scores_std = np.std(valid_scores, axis=1)


xlabel = '# of Neighbors'
ylabel = 'Accuracy'

plt_title = 'Validation Curve, KNN'

fig = plt.figure()
ax = fig.add_subplot(111, xlabel=xlabel, ylabel=ylabel, title=plt_title)
plt.semilogx(n_neighbors_list, train_scores_mean, label="Training score",
             color="darkorange", lw=2)
plt.fill_between(n_neighbors_list, train_scores_mean - train_scores_std,
                 train_scores_mean + train_scores_std, alpha=0.2,
                 color="darkorange", lw=2)
plt.semilogx(n_neighbors_list, valid_scores_mean, label="Validation score",
             color="navy", lw=2)
plt.fill_between(n_neighbors_list, valid_scores_mean - valid_scores_std,
                 valid_scores_mean + valid_scores_std, alpha=0.2,
                 color="navy", lw=2)
plt.legend(loc="best")
plt.savefig('results/validation_curve_KNN.jpg', bbox_inches='tight', dpi=200)
plt.show()
#learning curve
from sklearn.model_selection import learning_curve

n_neighbors_list = range(3, 15)

ns_list, train_scores, validation_scores = learning_curve(
                                                   estimator = neighbors.KNeighborsClassifier(metric='l1', n_jobs=4), 
                                                   X = X_train, y = Y_train, 
                                                   train_sizes = n_neighbors_list, 
                                                   scoring = 'accuracy', 
                                                   n_jobs= -1
                                                   )

train_scores_mean =  train_scores.mean(axis = 1)
train_scores_std = np.std(train_scores, axis=1)
validation_scores_mean =  validation_scores.mean(axis = 1)
validation_scores_std  = validation_scores.std(axis = 1)

xlabel = '# of Neighbors'
ylabel = 'Accuracy'

plt_title = 'Learning Curve, KNN'

fig = plt.figure()
ax = fig.add_subplot(111, xlabel=xlabel, ylabel=ylabel, title=plt_title)
ax.plot(n_neighbors_list, train_scores_mean, label = 'Training Score')
ax.plot(n_neighbors_list, validation_scores_mean, label = 'Validation Score')

plt.fill_between(n_neighbors_list, train_scores_mean - train_scores_std,
                 train_scores_mean + train_scores_std, alpha=0.2,
                 color="darkblue", lw=2)
plt.fill_between(n_neighbors_list, validation_scores_mean - validation_scores_std,
                 validation_scores_mean + validation_scores_std, alpha=0.2,
                 color="darkgreen", lw=2)


ax.legend(loc=0)
plt.savefig("results/learning_curve_KNN.jpg", bbox_inches='tight', dpi=200)
plt.show()<jupyter_output>/home/phoenix/Apps/anaconda3/envs/physics/lib/python3.9/site-packages/sklearn/model_selection/_validation.py:771: UserWarning: Scoring failed. The score on this train-test partition for these parameters will be set to nan. Details: 
Traceback (most recent call last):
  File "/home/phoenix/Apps/anaconda3/envs/physics/lib/python3.9/site-packages/sklearn/model_selection/_validation.py", line 762, in _score
    scores = scorer(estimator, X_test, y_test)
  File "/home/phoenix/Apps/anaconda3/envs/physics/lib/python3.9/site-packages/sklearn/metrics/_scorer.py", line 216, in __call__
    return self._score(
  File "/home/phoenix/Apps/anaconda3/envs/physics/lib/python3.9/site-packages/sklearn/metrics/_scorer.py", line 258, in _score
    y_pred = method_caller(estimator, "predict", X)
  File "/home/phoenix/Apps/anaconda3/envs/physics/lib/python3.9/site-packages/sklearn/metrics/_scorer.py", line 68, in _cached_call
    return getattr(estimator, method)(*args, **kwargs)
  File "/home/phoenix/Apps/[...]<jupyter_text>MetricsCouldn't think of any other mectrics<jupyter_code># confusion metrics
from sklearn.metrics import confusion_matrix
print("Decision Tree:\n", confusion_matrix(y_true=Y_test, y_pred=dt_clf.predict(X_test), normalize='true', labels=[1, 2, 3]))
print("\n\n")
print("NCA Pipeline:\n", confusion_matrix(y_true=Y_test, y_pred=nca_pipe.predict(X_test), normalize='true', labels=[1, 2, 3]))<jupyter_output>Decision Tree:
 [[0.9132948  0.06936416 0.01734104]
 [0.31410256 0.53846154 0.1474359 ]
 [0.00929752 0.02892562 0.96177686]]



NCA Pipeline:
 [[0.9132948  0.05780347 0.02890173]
 [0.35897436 0.47435897 0.16666667]
 [0.02169421 0.02995868 0.94834711]]
{'fraction_non_alphanumeric': 0.10799285997619992, 'fraction_numerical': 0.032625942086473625, 'mean_word_length': 3.027555910543131, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '367151', 'license': 'MIT', 'n_tokens_mistral': 3641, 'n_tokens_neox': 3276, 'n_words': 559, 'path': 'Phase2/knn-dt.ipynb', 'repo_name': 'ali-mahani/finding-criticality', 'size': '10084'}