<jupyter_start><jupyter_text>Preprocessing*base on the work of Raymond*Use  ``` CountVectorizer ``` and ``` TfidfVectorizer ``` to extract features<jupyter_code>train_x = train_df.article_words
train_y = train_df.topic
test_x = test_df.article_words
test_y = test_df.topic

en_train_y = preprocessing.LabelEncoder().fit_transform(train_y)
en_test_y = preprocessing.LabelEncoder().fit_transform(test_y)
categories_list=list(set(train_df['topic']))
print(categories_list)
def convert_sparse_matrix_to_sparse_tensor(X):
    coo = X.tocoo()
    indices = np.mat([coo.row, coo.col]).transpose()
    sp = tf.SparseTensor(indices, coo.data, coo.shape)
    sp = tf.sparse.reorder(sp)
    return sp
# get Count vectors
vectorizer = CountVectorizer().fit(train_x)
train_count_csr = vectorizer.transform(train_x)
test_count_csr = vectorizer.transform(test_x)

# change count vectors from scipy.sparse.csr.csr_matrix to tf.sparse.SparseTensor
train_count_st = convert_sparse_matrix_to_sparse_tensor(train_count_csr)
test_count_st = convert_sparse_matrix_to_sparse_tensor(test_count_csr)

# change count vectors from tf.sparse.SparseTensor to tf.Tensor
train_count_dense = tf.sparse.to_dense(train_count_st)
test_count_dense = tf.sparse.to_dense(test_count_st)

# get Tfid vectors
vectorizer = TfidfVectorizer().fit(train_x)
train_tfid_csr = vectorizer.transform(train_x)
test_tfid_csr = vectorizer.transform(test_x)

# change Tfid vectors from scipy.sparse.csr.csr_matrix to tf.sparse.SparseTensor
train_tfid_st = convert_sparse_matrix_to_sparse_tensor(train_tfid_csr)
test_tfid_st = convert_sparse_matrix_to_sparse_tensor(test_tfid_csr)

# change Tfid vecotors from tf.sparse.SparseTensor to tf.Tensor
train_tfid_dense = tf.sparse.to_dense(train_tfid_st)
test_tfid_dense = tf.sparse.to_dense(test_tfid_st)

# # get Tfid vector vocab size
# vocab_size_tfid = train_vectors.shape[1]
# print(train_vectors.shape,test_vectors.shape)<jupyter_output><empty_output><jupyter_text>Preprocessing data and preserve orderFrom ```Tokenizer``` to ```pad_sequences```<jupyter_code>word_num = 500
tokenizer = Tokenizer(num_words=word_num)
tokenizer.fit_on_texts(train_x)
train_token = tokenizer.texts_to_sequences(train_x)
test_token = tokenizer.texts_to_sequences(test_x)
train_pad = pad_sequences(train_token)
word_length = train_pad.shape[1]
test_pad = pad_sequences(test_token, maxlen=word_length)
def cate_accuracies(matrix, categories):
    for i_row in range(len(matrix)):
        acc = matrix[i_row][i_row]/sum(matrix[i_row])
        print(f"The accuracy for {categories[i_row]} is {acc}.")<jupyter_output><empty_output><jupyter_text>Training Model 1  Best Training model so farAlready tried:* other linear layers combinations     * with/without dropout layers    * with/without weight decay* tokenize + embedding + LSTM    * extremely slow and inaccurate, not sure if it is because of underfitting or context information not preserved since it is too slow<jupyter_code>def network():
    model = Sequential()
    model.add(Dense(128, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(64, activation='relu'))
    model.add(Dropout(0.5))
    model.add(Dense(11, activation='relu'))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model

estimator = KerasClassifier(build_fn=network, epochs=50, batch_size=20, verbose=0)

start_time = time.time()
estimator.fit(train_count_st, train_y)
print(f"Training time {time.time()-start_time:.0f}s")
train_pred = estimator.predict(train_count_st)
print("Training confusion matrix")
print(metrics.confusion_matrix(train_y, train_pred, categories_list))

train_f1 = metrics.f1_score(train_y, train_pred, average='macro')
print(f"Training f1 score is {train_f1}.")

train_acc = metrics.accuracy_score(train_y, train_pred)
print(f"Training accuracy score is {train_acc}.")

train_recall = metrics.recall_score(train_y, train_pred, average="macro")
print(f"Training recall score is {train_recall}")

test_pred = estimator.predict(test_count_st)
print("Test confusion matrix")

confusion = metrics.confusion_matrix(test_y, test_pred, categories_list)
print(confusion)

test_f1 = metrics.f1_score(test_y, test_pred, average='macro')
print(f"Test f1 score is {test_f1}.")

test_acc = metrics.accuracy_score(test_y, test_pred)
print(f"Test accuracy score is {test_acc}.")

test_recall = metrics.recall_score(test_y, test_pred, average="macro")
print(f"Test recall score is {test_recall}")

test_report = metrics.classification_report(test_y, test_pred)
print(test_report)

cate_accuracies(confusion, categories_list)
def network():
    model = Sequential()
    model.add(Dense(128, activation='relu',
              kernel_regularizer=l2(0.01),
              bias_regularizer=l2(0.01)))
    model.add(Dropout(0.5))
    model.add(Dense(64, activation='relu',
              kernel_regularizer=l2(0.01),
              bias_regularizer=l2(0.01)))
    model.add(Dropout(0.5))
    model.add(Dense(11, activation='relu',
              kernel_regularizer=l2(0.01),
              bias_regularizer=l2(0.01)))
    model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
    return model


estimator = KerasClassifier(build_fn=network, epochs=10, batch_size=20, verbose=0)

start_time = time.time()
estimator.fit(train_count_st, train_y)
print(f"Training time {time.time()-start_time:.0f}s")
train_pred = estimator.predict(train_count_st)
print("Training confusion matrix")
print(metrics.confusion_matrix(train_y, train_pred, categories_list))

train_f1 = metrics.f1_score(train_y, train_pred, average='macro')
print(f"Training f1 score is {train_f1}.")

train_acc = metrics.accuracy_score(train_y, train_pred)
print(f"Training accuracy score is {train_acc}.")

train_recall = metrics.recall_score(train_y, train_pred, average="macro")
print(f"Training recall score is {train_recall}")

test_pred = estimator.predict(test_count_st)
print("Test confusion matrix")

confusion = metrics.confusion_matrix(test_y, test_pred, categories_list)
print(confusion)

test_f1 = metrics.f1_score(test_y, test_pred, average='macro')
print(f"Test f1 score is {test_f1}.")

test_acc = metrics.accuracy_score(test_y, test_pred)
print(f"Test accuracy score is {test_acc}.")

test_recall = metrics.recall_score(test_y, test_pred, average="macro")
print(f"Test recall score is {test_recall}")

test_report = metrics.classification_report(test_y, test_pred)
print(test_report)

cate_accuracies(confusion, categories_list)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1144126206157145, 'fraction_numerical': 0.012253024965538368, 'mean_word_length': 6.811004784688995, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '151987', 'license': 'Apache-2.0', 'n_tokens_mistral': 2369, 'n_tokens_neox': 2295, 'n_words': 405, 'path': 'Brad_work.ipynb', 'repo_name': 'HenryLiangzy/COMP9417_Project', 'size': '6529'}