<jupyter_start><jupyter_text>Read in Data and Preprocess<jupyter_code>true_news_df = pd.read_csv("COVID19-FNIR/COVID19-FNIR/trueNews.csv")
fake_news_df = pd.read_csv("COVID19-FNIR/COVID19-FNIR/fakeNews.csv")
# Assign class columns and combine dataframes
fake_news_df["class"] = 1
true_news_df["class"] = 0
combined = pd.concat([fake_news_df, true_news_df], axis=0)
fake_news_df[fake_news_df["Text"].str.contains("<NAME> is working")]["Text"].values
 # <NAME> is working on an electronic chip to implant under human skin in order to locate and control people.\t\t
true_news_df[true_news_df["Text"].str.contains("beyond recognition")]["Text"].values
# 'Covid-19 can damage lungs of victims beyond recognition, expert says  https://www.theguardian.com/world/2020/jun/15/covid-19-can-damage-lungs-victims-beyond-recognition-expert-says?utm_term=Autofeed&CMP=twt_b-gdnnews&utm_medium=Social&utm_source=Twitter#Echobox=1592245639\xa0'],
# Subset data to just contain the text field and the class field
combined = combined[["Text", "class"]].reset_index(drop=True)
# Shuffle data
combined = combined.sample(frac = 1, random_state=42)
combined['Text'] = combined['Text'].str.replace('http\S+|www.\S+', '', case=False)
combined['Text'] = combined['Text'].str.replace('pic\S+|twitter.\S+', '', case=False)
combined['Text'] = combined['Text'].str.replace('#\S+', '', case=False)
combined['Text'] = combined['Text'].str.replace('@\S+', '', case=False)
combined['Text'] = combined['Text'].str.replace('Link:\S+', '', case=False)
combined['Text'] = combined['Text'].str.replace('[ \t]+$', '', case=False)
# remove front and ending blank spaces
combined = combined.replace({"^\s*|\s*$":""}, regex=True) 

def format_text(text):
    """Formating of the text field """
    text = text.lower()
    text = re.sub('\[.*?\]', '', text)
    text = re.sub("\\W"," ",text) 
    text = re.sub('<.*?>+', '', text)
    text = re.sub('[%s]' % re.escape(string.punctuation), '', text)
    text = re.sub('\n', '', text)
    text = re.sub('\w*\d\w*', '', text)    
    text = re.sub(' +', ' ', text)
    return text

combined["Text"] = combined["Text"].apply(format_text)
# Set X and y variables
X = combined["Text"]
y = combined["class"]
# Split data into training and test dataset
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)
# Apply TDidf Vectorization to the input data 
vectorization = TfidfVectorizer()
X_train = vectorization.fit_transform(X_train)
joblib.dump(vectorization, 'tfidf_vectorizer.pkl')
vectorization = joblib.load('tfidf_vectorizer.pkl')
X_test = vectorization.transform(X_test)<jupyter_output><empty_output><jupyter_text>Model 1: Naive Bayes Model<jupyter_code># Default hyper-parameter model
nb_model = MultinomialNB()
# Fit model 
nb_model.fit(X_train, y_train)
# Predict 
y_pred = nb_model.predict(X_test)
print(f"Test Accuracy: {accuracy_score(y_test, y_pred)}")
print(classification_report(y_test, y_pred))
plot_conf_matrix(y_pred, y_test, title="Naive Bayes")
# GridSearch Across hyper-parameter search space
params = {
    "alpha": [0.001, 0.01, 0.05, 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1.0, 1.5, 2, 3, 5, 10],
    "fit_prior": (True, False),
}
gs_clf = GridSearchCV(MultinomialNB(), params, n_jobs=-1)
gs_clf = gs_clf.fit(X_train, y_train)
# Save Naive Bayes Model 
joblib.dump(gs_clf.best_estimator_, 'naive_bayes.pkl')
best_nb_model = joblib.load('naive_bayes.pkl')
y_pred = best_nb_model.predict(X_test)
print(f"Test Accuracy: {accuracy_score(y_test, y_pred)}")
print(f"Test Precision: {precision_score(y_test, y_pred)}")
print(f"Test Recall: {recall_score(y_test, y_pred)}")
print(f"Test f1 Score: {f1_score(y_test, y_pred)}")
print(f"Test roc auc score: {roc_auc_score(y_test, y_pred)}")
print(classification_report(y_test, y_pred))
plot_conf_matrix(y_pred, y_test, title="Naive Bayes")<jupyter_output><empty_output><jupyter_text>Model 2: Gradient Boosting Classifier<jupyter_code>gbt_model = GradientBoostingClassifier()
# Fit model 
gbt_model.fit(X_train, y_train)
# Predict 
y_pred = gbt_model.predict(X_test)
print(f"Test Accuracy: {accuracy_score(y_test, y_pred)}")
print(f"Test Precision: {precision_score(y_test, y_pred)}")
print(f"Test Recall: {recall_score(y_test, y_pred)}")
print(f"Test f1 Score: {f1_score(y_test, y_pred)}")
print(f"Test roc auc score: {roc_auc_score(y_test, y_pred)}")
print(classification_report(y_test, y_pred))
plot_conf_matrix(y_pred, y_test, title="Gradient Boosting Classifier Classifier")
%%time
params = {
    "loss": ["deviance", "exponential"],
    "learning_rate": [0.001, 0.01, 0.1, 1, 2],
    "n_estimators": [10, 50, 100, 500],
    "subsample": [0.5, 0.75, 1.0],
}
gs_clf = GridSearchCV(GradientBoostingClassifier(), params, n_jobs=-1, scoring='accuracy')
  
gs_clf = gs_clf.fit(X_train, y_train)
# Save Naive Bayes Model 
joblib.dump(gs_clf.best_estimator_, 'gradient_boosting.pkl')
best_gb_model = joblib.load('gradient_boosting.pkl')
# Predict 
y_pred = best_gb_model.predict(X_test)
print(f"Test Accuracy: {accuracy_score(y_test, y_pred)}")
print(f"Test Precision: {precision_score(y_test, y_pred)}")
print(f"Test Recall: {recall_score(y_test, y_pred)}")
print(f"Test f1 Score: {f1_score(y_test, y_pred)}")
print(f"Test roc auc score: {roc_auc_score(y_test, y_pred)}")
print(classification_report(y_test, y_pred))
plot_conf_matrix(y_pred, y_test, title="Gradient Boosting Classifier Classifier")<jupyter_output><empty_output><jupyter_text>Bidirectional LSTM<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)
X_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.25, random_state=42)

X_train = np.array(X_train)
y_train = np.array(y_train)

X_val = np.array(X_val)
y_val = np.array(y_val)

X_test = np.array(X_test)
y_test = np.array(y_test)
# Perform parameter sweep using Weights and Biases

wandb.init(project="my-test-project", entity="tripleblackcat")

sweep_config = {
    "name": "b-lstm",
    "metric": {"name": "val_f1", "goal": "maximize"},
    "method": "random",
    "parameters": {
        "epochs": {"values": [10, 20, 40]},
        "lr": {"values": [0.005, 0.001, 0.0005, 0.0001]},
        "batch_size": {"values": [64, 128, 256]},
        "max_tokens": {"values": [100, 1000, 5000, None]},
        "embedding_dim": {"values": [16, 32, 64, 256]},
        "lstm_units": {"values": [16, 32, 64, 256]},
        "ff_size": {"values": [64, 128, 256]},
        "dropout": {"values": [0, 0.5]}
    }
}
def get_lstm(config, encoder, vocab_size):
    LSTM = tf.keras.Sequential([
    encoder,
    tf.keras.layers.Embedding(
        input_dim=vocab_size,
        output_dim=config.embedding_dim,
        mask_zero=True),
    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(config.lstm_units)),
    tf.keras.layers.Dense(config.ff_size, activation="relu"),
    tf.keras.layers.Dropout(config.dropout),
    tf.keras.layers.Dense(1)
    ])
    return LSTM

def sweep():
    with wandb.init() as run:
        config = wandb.config 

        max_tokens = config.max_tokens
        encoder = tf.keras.layers.TextVectorization(max_tokens=max_tokens)
        encoder.adapt(X_train)

        vocab_size = len(encoder.get_vocabulary())

        lstm = get_lstm(config, encoder, vocab_size)
        lstm.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True), 
                     optimizer=tf.keras.optimizers.Adam(config.lr),
                     metrics=["accuracy"])

        lstm.fit(X_train, 
                 y_train, 
                 validation_data=(X_val, y_val), 
                 epochs=config.epochs,
                 callbacks=[WandbCallback()])
       
        y_pred = np.around(tf.math.sigmoid(lstm.predict(X_test))).flatten()
        wandb.log({"val_f1": f1_score(y_test, y_pred)})
        del lstm

sweep_id = wandb.sweep(sweep_config)
wandb.agent(sweep_id, sweep, count=50)
# https://wandb.ai/tripleblackcat/uncategorized/sweeps/aoizs7rc/table?workspace=user-tripleblackcat
# Train multiple times and average out results
# Use best model parameters from sweep
accuracies = []
precisions = []
recalls = []
f1s = []

runs = 10

max_tokens = None
encoder = tf.keras.layers.TextVectorization(max_tokens=max_tokens)
encoder.adapt(X_train)
vocab_size = len(encoder.get_vocabulary())
for i in range(runs):
    model = tf.keras.Sequential([
        encoder,
        tf.keras.layers.Embedding(
            input_dim=vocab_size,
            output_dim=32,
            mask_zero=True),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)),
        tf.keras.layers.Dense(64, activation="relu"),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Dense(1)
    ])
    model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True), 
              optimizer=tf.keras.optimizers.Adam(1e-4),
              metrics=["accuracy"])
    history = model.fit(X_train, y_train, validation_data=(X_val, y_val), epochs=10, batch_size=64)
    y_pred = np.around(tf.math.sigmoid(model.predict(X_test))).flatten()
    
    accuracies.append(accuracy_score(y_test, y_pred))
    precisions.append(precision_score(y_test, y_pred))
    recalls.append(recall_score(y_test, y_pred))
    f1s.append(f1_score(y_test, y_pred))
print(f"Test Accuracy: {np.array(accuracies).sum()/runs}")
print(f"Test Precision: {np.array(precisions).sum()/runs}")
print(f"Test Recall: {np.array(recalls).sum()/runs}")
print(f"Test F1 Score: {np.array(f1s).sum()/runs}")
# Saving via model.save() and load_model was not working
# Had to save model weights and load explicitly
def create_model():
    model = tf.keras.Sequential([
        encoder,
        tf.keras.layers.Embedding(
            input_dim=vocab_size,
            output_dim=32,
            mask_zero=True),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)),
        tf.keras.layers.Dense(64, activation="relu"),
        tf.keras.layers.Dropout(0.5),
        tf.keras.layers.Dense(1)
    ])
    model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True), 
              optimizer=tf.keras.optimizers.Adam(1e-4),
              metrics=["accuracy"])
    return model
# Save the weights
model.save_weights('bilstm/my_checkpoint')

# Create a new model instance
model = create_model()

# Restore the weights
model.load_weights('bilstm/my_checkpoint')
y_pred = np.around(tf.math.sigmoid(model.predict(X_test))).flatten()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1587377252359615, 'fraction_numerical': 0.022499761655067212, 'mean_word_length': 4.585729499467519, 'pattern_counts': {'":': 30, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '572485', 'license': 'MIT', 'n_tokens_mistral': 4166, 'n_tokens_neox': 3942, 'n_words': 671, 'path': 'Covid-Fake-News.ipynb', 'repo_name': 'ThishenP/covid-19-fake-news-detection', 'size': '10497'}