<jupyter_start><jupyter_text>Train Test Split  Stratified Split<jupyter_code>"""
from sklearn.model_selection import StratifiedShuffleSplit
split = StratifiedShuffleSplit(n_splits=1, test_size=0.4, random_state=42)

for train_index, test_index in split.split(df, df['AgeClass']):
    strat_train = df.loc[train_index]
    strat_test = df.loc[test_index]
    
# print(strat_test['AgeClass'].value_counts() / len(strat_test))
print(strat_test['AgeClass'].value_counts())
# print(strat_train['AgeClass'].value_counts() / len(strat_train))
print(strat_train['AgeClass'].value_counts())
# print(df['AgeClass'].value_counts() / len(df))
"""
"""
X_train = strat_train.drop(['Sex', 'Rings', 'Age', 'AgeClass'], axis=1)
y_train = strat_train['AgeClass']
y_train = pd.get_dummies(y_train)
X_test = strat_test.drop(['Sex', 'Rings', 'Age', 'AgeClass'], axis=1);
y_test = strat_test['AgeClass']
y_test = pd.get_dummies(y_test)
"""
# X_test, y_test
"""
X = df.drop(['Sex', 'Rings', 'Age', 'AgeClass'], axis=1)
y = df['AgeClass']
#X = df.drop(['Rings', 'Age', 'AgeClass'], axis=1)
#X = np.array(X)
# if using 'categorical_crossentropy' loss, then labels in one_hot representation
y = pd.get_dummies(y)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.4, random_state=42)  # stratify=y,  
"""<jupyter_output><empty_output><jupyter_text>Use Train, Test, Valid<jupyter_code>X = df.drop(['Sex', 'Rings', 'Age', 'AgeClass'], axis=1)
y = df['AgeClass']
# if using 'categorical_crossentropy' loss, then labels in one_hot representation
y = pd.get_dummies(y)
X_train, X_test, X_valid, y_train, y_test, y_valid = train_test_valid(X, y)
X_train.shape, y_train.shape, X_valid.shape, y_valid.shape, X_test.shape, y_test.shape, type(X_test), type(y_test)<jupyter_output><empty_output><jupyter_text>Scaling<jupyter_code>"""
from sklearn.preprocessing import StandardScaler, MinMaxScaler
#scaler = StandardScaler()
scaler = MinMaxScaler()

X = data.drop(['Sex', 'Rings', 'Age', 'AgeClass'], axis=1)
X_scaled = scaler.fit_transform(X)
y = data['AgeClass']
#X = df.drop(['Rings', 'Age', 'AgeClass'], axis=1)
#X = np.array(X)
# if using 'categorical_crossentropy' loss, then labels in one_hot representation
y = pd.get_dummies(y)
X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, test_size=0.4, random_state=42)
"""<jupyter_output><empty_output><jupyter_text>PCA Analysis<jupyter_code>"""
from sklearn.decomposition import PCA

X = df.drop(['Sex', 'Rings', 'Age', 'AgeClass'], axis=1)
y = df['AgeClass']
#X = df.drop(['Rings', 'Age', 'AgeClass'], axis=1)
#X = np.array(X)
# if using 'categorical_crossentropy' loss, then labels in one_hot representation
y = pd.get_dummies(y)


pca = PCA()
pca.fit(X)
print(pca.explained_variance_ratio_)
cumsum = np.cumsum(pca.explained_variance_ratio_)
d = np.argmax(cumsum >= 0.95) + 1
print(d)
X_pca = pca.transform(X)
X_train, X_test, y_train, y_test = train_test_split(X_pca, y, test_size=0.4, random_state=42)
"""<jupyter_output><empty_output><jupyter_text>Randomised Search CV<jupyter_code>param_distribs = {
    "n_hidden": np.arange(1,8),
    "n_neurons": np.arange(8, 64, 8),
    "learning_rate": reciprocal(1e-5, 1e-2),
}
keras_reg = keras.wrappers.scikit_learn.KerasClassifier(build_model)
rnd_search_cv = RandomizedSearchCV(keras_reg, param_distribs, n_iter=5, cv=2)
rnd_search_cv.fit(X_train, y_train, epochs=10,
                  validation_data=(X_valid, y_valid))

# model = rnd_search_cv.best_estimator_.model
print(rnd_search_cv.best_params_)
model = build_model(**rnd_search_cv.best_params_)
# Fit model
history = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=200, verbose=0)  # class_weight=?

print(rnd_search_cv.best_params_)
print(rnd_search_cv.best_score_)<jupyter_output><empty_output><jupyter_text>Set Search<jupyter_code># Find best learning rate
model_1_0 = build_model(n_hidden=1, n_neurons=16, learning_rate=8e-4, input_shape=None)
model_1_1 = build_model(n_hidden=1, n_neurons=16, learning_rate=1e-3, input_shape=None)
model_1_2 = build_model(n_hidden=1, n_neurons=16, learning_rate=2e-3, input_shape=None)
model_1_3 = build_model(n_hidden=1, n_neurons=16, learning_rate=4e-3, input_shape=None)
model_1_4 = build_model(n_hidden=1, n_neurons=16, learning_rate=6e-3, input_shape=None)
model_1_5 = build_model(n_hidden=1, n_neurons=16, learning_rate=8e-3, input_shape=None)
model_1_6 = build_model(n_hidden=1, n_neurons=16, learning_rate=1e-2, input_shape=None)
model_1_7 = build_model(n_hidden=1, n_neurons=16, learning_rate=2e-2, input_shape=None)  # **
model_1_8 = build_model(n_hidden=1, n_neurons=16, learning_rate=4e-2, input_shape=None)
model_1_9 = build_model(n_hidden=1, n_neurons=16, learning_rate=6e-2, input_shape=None)

# With best learning rate, find best number of hidden neurons
model_2_1 = build_model(n_hidden=1, n_neurons=16, learning_rate=2e-2, input_shape=None)  # **
model_2_2 = build_model(n_hidden=2, n_neurons=16, learning_rate=2e-2, input_shape=None)
model_2_3 = build_model(n_hidden=3, n_neurons=16, learning_rate=2e-2, input_shape=None)
model_2_4 = build_model(n_hidden=4, n_neurons=16, learning_rate=2e-2, input_shape=None)
model_2_5 = build_model(n_hidden=5, n_neurons=16, learning_rate=2e-2, input_shape=None) 
model_2_6 = build_model(n_hidden=6, n_neurons=16, learning_rate=2e-2, input_shape=None)
model_2_7 = build_model(n_hidden=7, n_neurons=16, learning_rate=2e-2, input_shape=None)
model_2_8 = build_model(n_hidden=8, n_neurons=16, learning_rate=2e-2, input_shape=None)
# With best lr, n_hidden, find best number of neurons
"""
model_3_1 = build_model(n_hidden=1, n_neurons=8, learning_rate=2e-2, input_shape=None)
model_3_2 = build_model(n_hidden=1, n_neurons=16, learning_rate=2e-2, input_shape=None)
model_3_3 = build_model(n_hidden=1, n_neurons=24, learning_rate=2e-2, input_shape=None)
model_3_4 = build_model(n_hidden=1, n_neurons=32, learning_rate=2e-2, input_shape=None)
model_3_5 = build_model(n_hidden=1, n_neurons=40, learning_rate=2e-2, input_shape=None)
model_3_6 = build_model(n_hidden=1, n_neurons=48, learning_rate=2e-2, input_shape=None)
model_3_7 = build_model(n_hidden=1, n_neurons=56, learning_rate=2e-2, input_shape=None)
model_3_8 = build_model(n_hidden=1, n_neurons=60, learning_rate=2e-2, input_shape=None)
"""
model_3_1 = build_model(n_hidden=2, n_neurons=8, learning_rate=2e-2, input_shape=None)
model_3_2 = build_model(n_hidden=2, n_neurons=16, learning_rate=2e-2, input_shape=None)
model_3_3 = build_model(n_hidden=2, n_neurons=24, learning_rate=2e-2, input_shape=None)
model_3_4 = build_model(n_hidden=2, n_neurons=32, learning_rate=2e-2, input_shape=None)
model_3_5 = build_model(n_hidden=2, n_neurons=40, learning_rate=2e-2, input_shape=None)  # **
model_3_6 = build_model(n_hidden=2, n_neurons=48, learning_rate=2e-2, input_shape=None)
model_3_7 = build_model(n_hidden=2, n_neurons=56, learning_rate=2e-2, input_shape=None)
model_3_8 = build_model(n_hidden=2, n_neurons=60, learning_rate=2e-2, input_shape=None)
models = [model_1_0, model_1_1, model_1_2, model_1_3, model_1_4,
          model_1_5, model_1_6, model_1_7, model_1_8, model_1_9]

model_names = ["model_1_0", "model_1_1", "model_1_2", "model_1_3", "model_1_4",
               "model_1_5", "model_1_6", "model_1_7", "model_1_8", "model_1_9"]
models = [model_2_1, model_2_2, model_2_3, model_2_4,
          model_2_5, model_2_6, model_2_7, model_2_8]

model_names = ["model_2_1", "model_2_2", "model_2_3", "model_2_4",
               "model_2_5", "model_2_6", "model_2_7", "model_2_8"]
models = [model_3_1, model_3_2, model_3_3, model_3_4,
          model_3_5, model_3_6, model_3_7, model_3_8]

model_names = ["model_3_1", "model_3_2", "model_3_3", "model_3_4",
               "model_3_5", "model_3_6", "model_3_7", "model_3_8"]
summary_dict = {}

for i in range(len(models)):
    models[i].fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
    loss, tp, fp, tn, fn, accuracy, precision, recall, auc, prc, *validations = models[i].evaluate(X_test, y_test)
    summary_dict[model_names[i]] = [loss, tp, fp, tn, fn, accuracy, precision, recall, auc, prc, model_names[i]]
    

summary=pd.DataFrame(summary_dict).transpose();
col_names={0:"loss", 1:"tp", 2:"fp", 3:"tn", 4:"fn", 5:"accuracy", 6:"precision", 7:"recall", 8:"auc", 9:"prcurve", 10:"model"}
summary.rename(columns = col_names, inplace=True)
summary['loss'] = summary['loss'].astype(float)
summary['accuracy'] = summary['accuracy'].astype(float)
summary['precision'] = summary['precision'].astype(float)
summary['recall'] = summary['recall'].astype(float)
summary['auc'] = summary['auc'].astype(float)
summary['prcurve'] = summary['prcurve'].astype(float)

summary['tp'] = summary['tp'].astype(int)
summary['fp'] = summary['fp'].astype(int)
summary['tn'] = summary['tn'].astype(int)
summary['fn'] = summary['fn'].astype(int)
summary.info()
summary
sns.set_theme(style='ticks')

fig, ((ax0, ax1), (ax2, ax3)) = plt.subplots(nrows=2, ncols=2, sharex=True,  figsize=(10,10))  # xticks(rotation=45),

ax0.plot(summary['tn'], 'o-')
ax0.set_title('True Negatives')
ax1.plot(summary['fp'], 'o-')
ax1.set_title('False Positives')
ax2.plot(summary['fn'], 'o-')
ax2.set_title('False Negatives')
ax3.plot(summary['tp'], 'o-')
ax3.set_title('True Positives')
fig.suptitle('Investigating Learning Rate on Neural Network model')
ax3.annotate('best model (#7)',
            xy=(615, 330), xycoords='figure points',
            xytext=(0, -125), textcoords='offset points',
            arrowprops=dict(facecolor='black', shrink=0.05),
            horizontalalignment='center', verticalalignment='bottom')
ax2.tick_params(labelrotation=45)
ax3.tick_params(labelrotation=45)
plt.tight_layout()
plt.savefig("./images/LR_h-paramater_tp-tn-fp-fn.png")
plt.show()
sns.set_theme(style='ticks')

fig, ((ax0, ax1), (ax2, ax3)) = plt.subplots(nrows=2, ncols=2, sharex=True,  figsize=(10,10))  # xticks(rotation=45),

ax0.plot(summary['tn'], 'o-')
ax0.set_title('True Negatives')
ax1.plot(summary['fp'], 'o-')
ax1.set_title('False Positives')
ax2.plot(summary['fn'], 'o-')
ax2.set_title('False Negatives')
ax3.plot(summary['tp'], 'o-')
ax3.set_title('True Positives')
fig.suptitle('Investigating hyperparameters on Neural Network model')
ax3.annotate('best model (#5)',
            xy=(600, 330), xycoords='figure points',
            xytext=(0, -125), textcoords='offset points',
            arrowprops=dict(facecolor='black', shrink=0.05),
            horizontalalignment='center', verticalalignment='bottom')
ax2.tick_params(labelrotation=45)
ax3.tick_params(labelrotation=45)
plt.tight_layout()
plt.savefig("./images/NNx_h-paramater_tp-tn-fp-fn.png")
plt.show()
history_1_1 = model_1_1.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)  # class_weight=?
history_1_2 = model_1_2.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)  # class_weight=?
history_1_3 = model_1_3.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)  # class_weight=?
history_1_4 = model_1_4.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)  # class_weight=?

history_2_1 = model_2_1.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_2_2 = model_2_2.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_2_3 = model_2_3.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_2_4 = model_2_4.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)

history_3_1 = model_3_1.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_3_2 = model_3_2.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_3_3 = model_3_3.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_3_4 = model_3_4.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)

history_1_5 = model_1_5.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)  # class_weight=?
history_1_6 = model_1_6.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)  # class_weight=?
history_1_7 = model_1_7.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)  # class_weight=?
history_1_8 = model_1_8.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)  # class_weight=?

history_2_5 = model_2_5.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_2_6 = model_2_6.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_2_7 = model_2_7.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_2_8 = model_2_8.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)

history_3_5 = model_3_5.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_3_6 = model_3_6.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_3_7 = model_3_7.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
history_3_8 = model_3_8.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=50, verbose=0)
# best
#model_1_1 = build_model(n_hidden=1, n_neurons=64, learning_rate=5e-4, input_shape=None)
#model_1_1 = build_model(n_hidden=4, n_neurons=64, learning_rate=1e-3, input_shape=None)
# {'learning_rate': 0.0006713362563809696, 'n_hidden': 4, 'n_neurons': 56}

# #============
# summary_dict = {}

# for i in range(len(models)):
#     models[i].fit(X_train, y_train, validation_data=(X_test, y_test), epochs=50, verbose=0)
#     loss, tp, fp, tn, fn, accuracy, precision, recall, auc, prc, *validations = models[i].evaluate(X_test, y_test)
#     summary_dict[model_names[i]] = [loss, tp, fp, tn, fn, accuracy, precision, recall, auc, prc, model_names[i]]
    

# summary=pd.DataFrame(summary_dict).transpose();
# col_names={0:"loss", 1:"tp", 2:"fp", 3:"tn", 4:"fn", 5:"accuracy", 6:"precision", 7:"recall", 8:"auc", 9:"prcurve", 10:"model"}
# summary.rename(columns = col_names, inplace=True)
#============

ten_times = {}

#model = build_model(n_hidden=4, n_neurons=56, learning_rate=7e-4, input_shape=None) # model_3_1
#model = build_model(n_hidden=2, n_neurons=16, learning_rate=8e-4, input_shape=None) # model_2_2
#model = build_model(n_hidden=2, n_neurons=40, learning_rate=2e-4, input_shape=None)  # ** 3_5
model = build_model(n_hidden=2, n_neurons=40, learning_rate=2e-2, input_shape=None)  # ** _3_5
for i in range(10):    
    model.fit(X_train, y_train, validation_data=(X_valid, y_valid), epochs=20, verbose=0)
    loss, tp, fp, tn, fn, accuracy, precision, recall, auc, prc, *validations = model.evaluate(X_train, y_train)
    ten_times[i] = [loss, tp, fp, tn, fn, accuracy, precision, recall, auc, prc]
    
ten_runs = pd.DataFrame(ten_times).transpose()
ten_runs.rename(columns=col_names, inplace=True)
ten_runs
ten_runs.describe()
#print(model.predict(X_test))
y_predict = np.argmax(model.predict(X_test), axis=1)+1
y_test_array = np.argmax(np.array(y_test), axis=1)+1
y_test_array
from sklearn.metrics import confusion_matrix
print(confusion_matrix(y_test_array, y_predict))
import seaborn as sns
def confusion_plot(y, y_pred):
    matrix = confusion_matrix(y, y_pred)
    #print(matrix)
    row_names = ['1', '2', '3', '4']
    col_names = ['1', '2', '3', '4']
    df_confusion = pd.DataFrame(matrix, index=row_names, columns=col_names)
    #print(df_confusion)
    plt.figure(figsize = (6,5))
    sns.heatmap(df_confusion, annot=True,  cmap="Blues", fmt=".0f", annot_kws={"size": 16})    
    sns.set(font_scale=1.4)#for label size  
    plt.ylim([0,4])
    plt.gca().invert_yaxis()
    plt.title("Confusion Matrix")
    plt.ylabel("True Class")
    plt.xlabel("Predicted Class")
    plt.savefig('./images/confusion_matrix.png')
    plt.show()
    plt.close()
    
confusion_plot(y_test_array, y_predict)
confusion = confusion_matrix(y_test_array, y_predict)
print('Confusion Matrix\n')
print(confusion)

#importing accuracy_score, precision_score, recall_score, f1_score
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
print('\nAccuracy: {:.2f}\n'.format(accuracy_score(y_test_array, y_predict)))

print('Micro Precision: {:.2f}'.format(precision_score(y_test_array, y_predict, average='micro')))
print('Micro Recall: {:.2f}'.format(recall_score(y_test_array, y_predict, average='micro')))
print('Micro F1-score: {:.2f}\n'.format(f1_score(y_test_array, y_predict, average='micro')))

print('Macro Precision: {:.2f}'.format(precision_score(y_test_array, y_predict, average='macro')))
print('Macro Recall: {:.2f}'.format(recall_score(y_test_array, y_predict, average='macro')))
print('Macro F1-score: {:.2f}\n'.format(f1_score(y_test_array, y_predict, average='macro')))

print('Weighted Precision: {:.2f}'.format(precision_score(y_test_array, y_predict, average='weighted')))
print('Weighted Recall: {:.2f}'.format(recall_score(y_test_array, y_predict, average='weighted')))
print('Weighted F1-score: {:.2f}'.format(f1_score(y_test_array, y_predict, average='weighted')))

from sklearn.metrics import classification_report
print('\nClassification Report\n')
print(classification_report(y_test_array, y_predict, target_names=['Class 1', 'Class 2', 'Class 3', 'Class 4']))
# # Plot history
# # set metrics=['accuracy']
# pd.DataFrame(history.history).plot(figsize=(8, 5))

# plt.grid(True)
# plt.gca().set_ylim(0, 1)
# plt.gca().set_xlim(0, len(history.epoch))
# plt.legend()
# plt.show()

# save_model(model, "best_params.h5")
# #old_model = load_model("testing.h5")
# #print(old_model.summary())
df_1_1 = pd.DataFrame(history_1_1.history)
df_1_2 = pd.DataFrame(history_1_2.history)
df_1_3 = pd.DataFrame(history_1_3.history)
df_1_4 = pd.DataFrame(history_1_4.history)
df_2_1 = pd.DataFrame(history_2_1.history)
df_2_2 = pd.DataFrame(history_2_2.history)
df_2_3 = pd.DataFrame(history_2_3.history)
df_2_4 = pd.DataFrame(history_2_4.history)
df_3_1 = pd.DataFrame(history_3_1.history)
df_3_2 = pd.DataFrame(history_3_2.history)
df_3_3 = pd.DataFrame(history_3_3.history)
df_3_4 = pd.DataFrame(history_3_4.history)

df_1_5 = pd.DataFrame(history_1_5.history)
df_1_6 = pd.DataFrame(history_1_6.history)
df_1_7 = pd.DataFrame(history_1_7.history)
df_1_8 = pd.DataFrame(history_1_8.history)
df_2_5 = pd.DataFrame(history_2_5.history)
df_2_6 = pd.DataFrame(history_2_6.history)
df_2_7 = pd.DataFrame(history_2_7.history)
df_2_8 = pd.DataFrame(history_2_8.history)
df_3_5 = pd.DataFrame(history_3_5.history)
df_3_6 = pd.DataFrame(history_3_6.history)
df_3_7 = pd.DataFrame(history_3_7.history)
df_3_8 = pd.DataFrame(history_3_8.history)

df_1_1['model'] = 11
df_1_2['model'] = 12
df_1_3['model'] = 13
df_1_4['model'] = 14
df_2_1['model'] = 21
df_2_2['model'] = 22
df_2_3['model'] = 23
df_2_4['model'] = 24
df_3_1['model'] = 31
df_3_2['model'] = 32
df_3_3['model'] = 33
df_3_4['model'] = 34

df_1_5['model'] = 15
df_1_6['model'] = 16
df_1_7['model'] = 17
df_1_8['model'] = 18
df_2_5['model'] = 25
df_2_6['model'] = 26
df_2_7['model'] = 27
df_2_8['model'] = 28
df_3_5['model'] = 35
df_3_6['model'] = 36
df_3_7['model'] = 37
df_3_8['model'] = 38

df_models = pd.concat([df_1_1, df_1_2, df_1_3, df_1_4, df_1_5, df_1_6, df_1_7, df_1_8,
                       df_2_1, df_2_2, df_2_3, df_2_4, df_2_5, df_2_6, df_2_7, df_2_8,
                       df_3_1, df_3_2, df_3_3, df_3_4, df_3_5, df_3_6, df_3_7, df_3_8,
                      ])
df_models['accuracy'].max()
df_1_1['accuracy'].plot(figsize=(8, 5))
df_1_2['accuracy'].plot()
df_1_3['accuracy'].plot()
df_1_4['accuracy'].plot()

plt.grid(True)
plt.gca().set_ylim(0, 1)
plt.gca().set_xlim(0, len(history_1_1.epoch))
plt.legend()
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1615958089865001, 'fraction_numerical': 0.052538787023977435, 'mean_word_length': 7.234342596433016, 'pattern_counts': {'":': 4, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '425589', 'license': 'MIT', 'n_tokens_mistral': 8590, 'n_tokens_neox': 8208, 'n_words': 1304, 'path': 'investigation.ipynb', 'repo_name': 'ABDunbar/NN_abalone', 'size': '19852'}