<jupyter_start><jupyter_text>Text Categorization With Neural Networks<jupyter_code>import numpy as np
import pandas as pd
import os
import random
import json
import tensorflow as tf
from bpmll import bp_mll_loss
from sklearn.model_selection import train_test_split
from sklearn import metrics
import sys
sys.path.append('../ThresholdFunctionLearning')    ## Append path to the ThresholdFunctionLearning directory to the interpreters
                                                   ## search path
from threshold_learning import predict_test_labels_binary    ## Import the 'predict_test_labels_binary()' function from the 
                                                             ## threshold_learning library<jupyter_output><empty_output><jupyter_text>Load & Pre-Process the Data<jupyter_code>## Load 'content_paragraphs_ready.csv' into a pandas dataframe
data_filepath = "..\..\dataset\content_paragraphs_ready.csv"
paragraph_data = pd.read_csv(data_filepath)
paragraph_data.head()
#os.getcwd()
## Keep only 'doc_id', the label columns, and the tf-idf columns
tfidf_colnames = list(paragraph_data.columns[25:])
label_columns = list(paragraph_data.columns[2:15])
cols_toKeep = ['doc_id']
cols_toKeep.extend(tfidf_colnames)
cols_toKeep.extend(label_columns)
paragraph_data = paragraph_data[cols_toKeep]
paragraph_data.head()
## Define the X and Y train and test matrices
X = paragraph_data[tfidf_colnames].to_numpy().astype(float)
Y = paragraph_data[label_columns].to_numpy().astype(float)

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.33, random_state = 321)
## Export train and test data to json file
tfidf_trainTest_data = {'X_train' : X_train.tolist(), 
                 'X_test' : X_test.tolist(),
                 'Y_train' : Y_train.tolist(), 
                 'Y_test' : Y_test.tolist()}

tfidf_trainTest_json = json.dumps(tfidf_trainTest_data)
with open("tfidf_trainTest_data.json", "w") as outfile: 
    json.dump(tfidf_trainTest_data, outfile)
## These are the rows that don't have any associated labels
Y_gz = Y > 0
no_labels_id = np.where(~Y_gz.any(axis=1))[0]
no_labels_id<jupyter_output><empty_output><jupyter_text>Define and Train a BP-MLL Network (Training Instances That Have Atleast One Label)<jupyter_code>## Since some instances don't have any labels, remove them from the dataset for use with BP-MLL
atleast_one_label_ids = paragraph_data.index
atleast_one_label_ids = atleast_one_label_ids.difference(no_labels_id)
X_hasLabel = X[atleast_one_label_ids, ]
Y_hasLabel = Y[atleast_one_label_ids, ]
X_train_hasLabel, X_test_hasLabel, Y_train_hasLabel, Y_test_hasLabel = train_test_split(X_hasLabel, Y_hasLabel, 
                                                                                        test_size = 0.33, random_state = 321)
Y_test_hasLabel
## Export reduced train and test data to json file
tfidf_trainTest_data_reduced = {'X_train_hasLabel' : X_train_hasLabel.tolist(), 
                 'X_test_hasLabel' : X_test_hasLabel.tolist(),
                 'Y_train_hasLabel' : Y_train_hasLabel.tolist(), 
                 'Y_test_hasLabel' : Y_test_hasLabel.tolist()}

tfidf_trainTest_data_reduced_json = json.dumps(tfidf_trainTest_data_reduced)
with open("tfidf_trainTest_data_reduced.json", "w") as outfile: 
    json.dump(tfidf_trainTest_data_reduced, outfile)
## Compute the proportion of labels that are equal to one in the test_hasLabel set
## This tells us what the hamming loss would be, if we constantly predicted 0
## We can compare our observed hamming loss against this, since we have sparse labels
prop_one_bpmll = np.sum(Y_test_hasLabel == 1) / (Y_test_hasLabel.shape[0] * Y_test_hasLabel.shape[1])
prop_one_bpmll
## Start by defining and compiling the bp-mll loss network 
num_labels = len(label_columns)

model_bpmll = tf.keras.models.Sequential([
    tf.keras.layers.Dense(32, activation = 'relu'),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(num_labels, activation = 'sigmoid')
])

optim_func_bpmll = tf.keras.optimizers.Adagrad(
    learning_rate = 0.001, initial_accumulator_value = 0.1, epsilon = 1e-07,
    name = 'Adagrad')

model_bpmll.compile(optimizer = optim_func_bpmll,
              loss = bp_mll_loss,
              )
tf.random.set_seed(123)
model_bpmll.fit(X_train_hasLabel, Y_train_hasLabel, epochs = 500)<jupyter_output>Epoch 1/100
6/6 [==============================] - 1s 2ms/step - loss: 0.9835
Epoch 2/100
6/6 [==============================] - 0s 1ms/step - loss: 0.9845
Epoch 3/100
6/6 [==============================] - 0s 2ms/step - loss: 0.9902
Epoch 4/100
6/6 [==============================] - 0s 2ms/step - loss: 0.9821
Epoch 5/100
6/6 [==============================] - 0s 2ms/step - loss: 0.9831
Epoch 6/100
6/6 [==============================] - 0s 2ms/step - loss: 0.9834
Epoch 7/100
6/6 [==============================] - 0s 2ms/step - loss: 0.9774
Epoch 8/100
6/6 [==============================] - 0s 1ms/step - loss: 0.9847
Epoch 9/100
6/6 [==============================] - 0s 2ms/step - loss: 0.9743
Epoch 10/100
6/6 [==============================] - 0s 2ms/step - loss: 0.9844
Epoch 11/100
6/6 [==============================] - 0s 3ms/step - loss: 0.9774
Epoch 12/100
6/6 [==============================] - 0s 2ms/step - loss: 0.9809
Epoch 13/100
6/6 [==============================] - 0s 1ms/st[...]<jupyter_text>Compare Performance of BP-MLL Loss Using Different Threshold Function Approaches<jupyter_code>## Using a constant 0.5 threshold function, get the hamming loss for the trained network on the test set
predictions = model_bpmll.predict(X_test_hasLabel)
predictions_binary = model_bpmll.predict(X_test_hasLabel)
for i in range(Y_test_hasLabel.shape[0]):
    for j in range(Y_test_hasLabel.shape[1]):
        if predictions_binary[i, j] > 0.5:
            predictions_binary[i, j] = 1
        else:
            predictions_binary[i, j] = 0

# Get the hamming loss
metrics.hamming_loss(Y_test_hasLabel, predictions_binary)
## Learn a Threshold Function
Y_train_pred = model_bpmll.predict(X_train_hasLabel)
Y_test_pred = model_bpmll.predict(X_test_hasLabel)
t_range = (0, 1)

test_labels_binary, threshold_function = predict_test_labels_binary(Y_train_pred, Y_train_hasLabel, Y_test_pred, t_range)
metrics.hamming_loss(Y_test_hasLabel, test_labels_binary)<jupyter_output><empty_output><jupyter_text>Deeper Architectures<jupyter_code>## Add an additional hidden layer
model_bpmll_deep = tf.keras.models.Sequential([
    tf.keras.layers.Dense(32, activation = 'relu'),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(16, activation = 'relu'),
    tf.keras.layers.Dense(num_labels, activation = 'sigmoid')
])

optim_func_bpmll = tf.keras.optimizers.Adagrad(
    learning_rate = 0.001, initial_accumulator_value = 0.1, epsilon = 1e-07,
    name = 'Adagrad')

model_bpmll_deep.compile(optimizer = optim_func_bpmll,
              loss = bp_mll_loss,
              )

model_bpmll_deep.fit(X_train_hasLabel, Y_train_hasLabel, epochs = 500)
## Using a constant 0.5 threshold function, get the hamming loss for the trained network on the test set
predictions = model_bpmll_deep.predict(X_test_hasLabel)
predictions_binary = model_bpmll_deep.predict(X_test_hasLabel)
for i in range(Y_test_hasLabel.shape[0]):
    for j in range(Y_test_hasLabel.shape[1]):
        if predictions_binary[i, j] > 0.5:
            predictions_binary[i, j] = 1
        else:
            predictions_binary[i, j] = 0

# Get the hamming loss
metrics.hamming_loss(Y_test_hasLabel, predictions_binary)
## Learn a Threshold Function
Y_train_pred = model_bpmll_deep.predict(X_train_hasLabel)
Y_test_pred = model_bpmll_deep.predict(X_test_hasLabel)
t_range = (0, 1)

test_labels_binary, threshold_function = predict_test_labels_binary(Y_train_pred, Y_train_hasLabel, Y_test_pred, t_range)
metrics.hamming_loss(Y_test_hasLabel, test_labels_binary)<jupyter_output><empty_output><jupyter_text>Define and Train a Cross-Entropy Loss Network (Training Instances That Have Atleast One Label)<jupyter_code>## Start with standard cross-entropy loss (bpmll used later)

model_ce_hasLabel = tf.keras.models.Sequential([
    tf.keras.layers.Dense(32, activation = 'relu'),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(num_labels, activation = 'sigmoid')
])

optim_func_ce = tf.keras.optimizers.Adagrad(
    learning_rate = 0.001, initial_accumulator_value = 0.1, epsilon = 1e-07,
    name = 'Adagrad')

model_ce_hasLabel.compile(optimizer = optim_func_ce,
              loss = 'categorical_crossentropy',
              )
model_ce_hasLabel.fit(X_train_hasLabel, Y_train_hasLabel, epochs = 500)<jupyter_output>Epoch 1/100
6/6 [==============================] - 0s 2ms/step - loss: 8.0969
Epoch 2/100
6/6 [==============================] - 0s 2ms/step - loss: 8.4308
Epoch 3/100
6/6 [==============================] - 0s 1ms/step - loss: 8.5960
Epoch 4/100
6/6 [==============================] - 0s 1ms/step - loss: 8.3554
Epoch 5/100
6/6 [==============================] - 0s 2ms/step - loss: 8.3493
Epoch 6/100
6/6 [==============================] - 0s 1ms/step - loss: 8.2851
Epoch 7/100
6/6 [==============================] - 0s 1ms/step - loss: 8.3467
Epoch 8/100
6/6 [==============================] - 0s 2ms/step - loss: 8.1551
Epoch 9/100
6/6 [==============================] - 0s 2ms/step - loss: 8.1740
Epoch 10/100
6/6 [==============================] - 0s 963us/step - loss: 8.1701
Epoch 11/100
6/6 [==============================] - 0s 2ms/step - loss: 8.3579
Epoch 12/100
6/6 [==============================] - 0s 1ms/step - loss: 8.2342
Epoch 13/100
6/6 [==============================] - 0s 1ms/[...]<jupyter_text>Compare Performance of CE Loss Using Different Threshold Function Approaches<jupyter_code>## Using a constant 0.5 threshold function, get the hamming loss for the trained network on the test set
predictions = model_ce_hasLabel.predict(X_test_hasLabel)
predictions_binary = model_ce_hasLabel.predict(X_test_hasLabel)
for i in range(Y_test_hasLabel.shape[0]):
    for j in range(Y_test_hasLabel.shape[1]):
        if predictions_binary[i, j] > 0.5:
            predictions_binary[i, j] = 1
        else:
            predictions_binary[i, j] = 0

# Get the hamming loss
metrics.hamming_loss(Y_test_hasLabel, predictions_binary)
## Learn a Threshold Function
Y_train_pred = model_ce_hasLabel.predict(X_train_hasLabel)
Y_test_pred = model_ce_hasLabel.predict(X_test_hasLabel)
t_range = (0, 1)

test_labels_binary, threshold_function = predict_test_labels_binary(Y_train_pred, Y_train_hasLabel, Y_test_pred, t_range)
metrics.hamming_loss(Y_test_hasLabel, test_labels_binary)<jupyter_output><empty_output><jupyter_text>Deeper Architectures<jupyter_code>## Add an additional hidden layer
model_ce_deep = tf.keras.models.Sequential([
    tf.keras.layers.Dense(32, activation = 'relu'),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(16, activation = 'relu'),
    tf.keras.layers.Dense(num_labels, activation = 'sigmoid')
])

optim_func_ce = tf.keras.optimizers.Adagrad(
    learning_rate = 0.001, initial_accumulator_value = 0.1, epsilon = 1e-07,
    name = 'Adagrad')

model_ce_deep.compile(optimizer = optim_func_ce,
              loss = 'categorical_crossentropy',
              )

model_ce_deep.fit(X_train_hasLabel, Y_train_hasLabel, epochs = 500)
## Using a constant 0.5 threshold function, get the hamming loss for the trained network on the test set
predictions = model_ce_hasLabel.predict(X_test_hasLabel)
predictions_binary = model_ce_hasLabel.predict(X_test_hasLabel)
for i in range(Y_test_hasLabel.shape[0]):
    for j in range(Y_test_hasLabel.shape[1]):
        if predictions_binary[i, j] > 0.5:
            predictions_binary[i, j] = 1
        else:
            predictions_binary[i, j] = 0

# Get the hamming loss
metrics.hamming_loss(Y_test_hasLabel, predictions_binary)
## Learn a Threshold Function
Y_train_pred = model_ce_deep.predict(X_train_hasLabel)
Y_test_pred = model_ce_deep.predict(X_test_hasLabel)
t_range = (0, 1)

test_labels_binary, threshold_function = predict_test_labels_binary(Y_train_pred, Y_train_hasLabel, Y_test_pred, t_range)
metrics.hamming_loss(Y_test_hasLabel, test_labels_binary)<jupyter_output><empty_output><jupyter_text>Define and Train a Cross-Entropy Loss Network (All Training Instances)<jupyter_code>## Compute the proportion of labels that are equal to one in the test set
## This tells us what the hamming loss would be, if we constantly predicted 0
## We can compare our observed hamming loss against this, since we have sparse labels
prop_one = np.sum(Y_test == 1) / (Y_test.shape[0] * Y_test.shape[1])
prop_one
## Start with standard cross-entropy loss (bpmll used later)
random.seed(123)
model_ce = tf.keras.models.Sequential([
    tf.keras.layers.Dense(32, activation = 'relu'),
    tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(num_labels, activation = 'sigmoid')
])

optim_func_ce = tf.keras.optimizers.Adagrad(
    learning_rate = 0.0001, initial_accumulator_value = 0.1, epsilon = 1e-07,
    name = 'Adagrad')

model_ce.compile(optimizer = optim_func_ce,
              loss = 'categorical_crossentropy',
              )
tf.random.set_seed(321)
model_ce.fit(X_train, Y_train, epochs = 500)<jupyter_output>Epoch 1/1000
7/7 [==============================] - 1s 2ms/step - loss: 7.8767
Epoch 2/1000
7/7 [==============================] - 0s 1ms/step - loss: 7.8061
Epoch 3/1000
7/7 [==============================] - 0s 1ms/step - loss: 7.6811
Epoch 4/1000
7/7 [==============================] - 0s 1ms/step - loss: 7.9014
Epoch 5/1000
7/7 [==============================] - 0s 2ms/step - loss: 7.6922
Epoch 6/1000
7/7 [==============================] - 0s 2ms/step - loss: 7.5552
Epoch 7/1000
7/7 [==============================] - 0s 2ms/step - loss: 7.9926
Epoch 8/1000
7/7 [==============================] - 0s 2ms/step - loss: 7.8372
Epoch 9/1000
7/7 [==============================] - 0s 1ms/step - loss: 7.8541
Epoch 10/1000
7/7 [==============================] - 0s 1ms/step - loss: 8.0131
Epoch 11/1000
7/7 [==============================] - 0s 2ms/step - loss: 7.5476
Epoch 12/1000
7/7 [==============================] - 0s 2ms/step - loss: 7.8630
Epoch 13/1000
7/7 [==============================[...]<jupyter_text>Compare Performance of CE Loss Using Different Threshold Function Approaches<jupyter_code>## Using a constant 0.5 threshold function, get the hamming loss for the trained network on the test set
predictions = model_ce.predict(X_test)
predictions_binary = model_ce.predict(X_test)
for i in range(Y_test.shape[0]):
    for j in range(Y_test.shape[1]):
        if predictions_binary[i, j] > 0.5:
            predictions_binary[i, j] = 1
        else:
            predictions_binary[i, j] = 0

# Get the hamming loss
metrics.hamming_loss(Y_test, predictions_binary)
## Learn a Threshold Function
Y_train_pred = model_ce.predict(X_train)
Y_test_pred = model_ce.predict(X_test)
t_range = (0, 1)

test_labels_binary, threshold_function = predict_test_labels_binary(Y_train_pred, Y_train, Y_test_pred, t_range)
metrics.hamming_loss(Y_test, test_labels_binary)
## Compare predicted test labels for instance against true test labels in next cell
test_labels_binary[4,]
## Compare true test labels for instance against predicted test labels in previous cell
Y_test[4,]<jupyter_output><empty_output><jupyter_text>Deeper Architectures<jupyter_code>## Standard cross-entropy loss 
random.seed(123)
model_ce_all_deep = tf.keras.models.Sequential([
    tf.keras.layers.Dense(64, activation = 'relu'),
    #tf.keras.layers.Dropout(0.5),
    tf.keras.layers.Dense(32, activation = 'relu'),
    #tf.keras.layers.Dropout(0.25),
    tf.keras.layers.Dense(32, activation = 'relu'),
    #tf.keras.layers.Dropout(0.25),
    tf.keras.layers.Dense(16, activation = 'relu'),
    tf.keras.layers.Dense(num_labels, activation = 'sigmoid')
])

optim_func_ce = tf.keras.optimizers.Adagrad(
    learning_rate = 0.0001, initial_accumulator_value = 0.1, epsilon = 1e-07,
    name = 'Adagrad')

model_ce_all_deep.compile(optimizer = optim_func_ce,
              loss = 'categorical_crossentropy',
              )
tf.random.set_seed(321)
model_ce_all_deep.fit(X_train, Y_train, epochs = 1000)
## Using a constant 0.5 threshold function, get the hamming loss for the trained network on the test set
predictions = model_ce_all_deep.predict(X_test)
predictions_binary = model_ce_all_deep.predict(X_test)
for i in range(Y_test.shape[0]):
    for j in range(Y_test.shape[1]):
        if predictions_binary[i, j] > 0.5:
            predictions_binary[i, j] = 1
        else:
            predictions_binary[i, j] = 0

# Get the hamming loss
metrics.hamming_loss(Y_test, predictions_binary)
## Learn a Threshold Function
Y_train_pred = model_ce_all_deep.predict(X_train)
Y_test_pred = model_ce_all_deep.predict(X_test)
t_range = (0, 1)

test_labels_binary, threshold_function = predict_test_labels_binary(Y_train_pred, Y_train, Y_test_pred, t_range)
metrics.hamming_loss(Y_test, test_labels_binary)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16808598108885886, 'fraction_numerical': 0.04328419568919951, 'mean_word_length': 4.535760728218466, 'pattern_counts': {'":': 0, '<': 42, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '362257', 'license': 'MIT', 'n_tokens_mistral': 6436, 'n_tokens_neox': 5824, 'n_words': 1294, 'path': 'codes/BP-MLL Text Categorization/BPMLL_paragraph_classification.ipynb', 'repo_name': 'architdatar/NewsArticleClassification', 'size': '17027'}