<jupyter_start><jupyter_text>Fake News Classification Connect Colab to VSCode<jupyter_code>!pip install kora -q
from kora import jupyter
jupyter.start(lab=True)
from google.colab import drive
drive.mount("/content/drive")
## Import Libraries
import tqdm
#@title Run this code to get started
imports = {
    #'pandas':'pd',
    'numpy': 'np',
    'matplotlib.pyplot': 'plt',
    'sklearn':None,
    'string':None,
    'nltk':None,
    'spacy':None,
    'wordcloud':None,
    'sys':None,
    'os':None}

print("Importing libraries")
for name in tqdm.tqdm(imports):
    try:
        alias = imports[name] or name
        globals()[alias] = __import__(name)
    except ImportError:
        print(f"Failed importing {name}")

# import math
# import os
# import numpy as np
# import pandas as pd

# import pickle

# import requests, io, zipfile

# Download class resources...

basepath = 'E:\\Desktop\\Dataset'
#basepath = 'drive/MyDrive/Notebooks/Fake_news_data'

from sklearn.linear_model import LogisticRegression
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
print("Importing NLP")
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
from spacy.lang.en.stop_words import STOP_WORDS
nltk.download('wordnet')
nltk.download('punkt')

from wordcloud import WordCloud
import matplotlib.pyplot as plt
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.metrics import accuracy_score
#!python -m spacy download en_core_web_md
import en_core_web_md
text_to_nlp = en_core_web_md.load()<jupyter_output>Importing NLP
[nltk_data] Downloading package wordnet to
[nltk_data]     C:\Users\lolzc\AppData\Roaming\nltk_data...
[nltk_data]   Package wordnet is already up-to-date!
[nltk_data] Downloading package punkt to
[nltk_data]     C:\Users\lolzc\AppData\Roaming\nltk_data...
[nltk_data]   Package punkt is already up-to-date!<jupyter_text>Dataset Views<jupyter_code>print("Topics:\n\t", *list(set(dataset['subject'])), sep='\n\t')

dataset.head()

dataset['date'].min()<jupyter_output><empty_output><jupyter_text>Generate Word cloud for fake/real news<jupyter_code>fake = True
print("Fake news Word cloud")
news_text = ''
for t in dataset[dataset['fake'] == fake]['text'].values: # form field cell
    news_text += t + ' '
    
wordcloud = WordCloud()    
wordcloud.generate_from_text(news_text)
plt.figure(figsize=(14,7))
plt.imshow(wordcloud, interpolation='bilinear')
fake = False
print("Real news Word cloud")
news_text = ''
for t in dataset[dataset['fake'] == fake]['text'].values: # form field cell
    news_text += t + ' '
    
wordcloud = WordCloud()    
wordcloud.generate_from_text(news_text)
plt.figure(figsize=(14,7))
plt.imshow(wordcloud, interpolation='bilinear')
fake = True
print("Fake news Title cloud")
news_text = ''
for t in dataset[dataset['fake'] == fake]['title'].values: # form field cell
    news_text += t + ' '
    
wordcloud = WordCloud()    
wordcloud.generate_from_text(news_text)
plt.figure(figsize=(14,7))
plt.imshow(wordcloud, interpolation='bilinear')
fake = False
print("True news Title cloud")
news_text = ''
for t in dataset[dataset['fake'] == fake]['title'].values: # form field cell
    news_text += t + ' '
    
wordcloud = WordCloud()    
wordcloud.generate_from_text(news_text)
plt.figure(figsize=(14,7))
plt.imshow(wordcloud, interpolation='bilinear')<jupyter_output><empty_output><jupyter_text>Prepare data for NLP<jupyter_code># SKIP
# if not "text_to_nlp" in globals():
#     print("Loading NLP converter")
#     text_to_nlp = en_core_web_md.load() #Prepare Spacy
# print("Tokenizing text")


# x_word2vec = np.zeros((len(dataset),), dtype=object)
# i = 0
# for text in tqdm.tqdm(dataset['text']):
#     x_word2vec[i] = text_to_nlp(text) # Array of tokens, passed to model
#     i += 1


# # print("Tokenizing titles")
# # x_titlevec = np.zeros((len(dataset),), dtype=object)
# # i = 0
# # for text in tqdm.tqdm(dataset['title']):
# #     x_titlevec[i] = text_to_nlp(text) # Array of tokens, passed to model
# #     i += 1
# # x_titlevec = np.array(x_word2vec)
# Memory mapper
print(*[f'{k} : {sys.getsizeof(v)} B' for (k, v) in globals().items()], sep='\n')
# x_word2vec = np.concat(np.load(os.path.join("E:\\Desktop\\Dataset", 'word_vecs1')), np.load(os.path.join("E:\\Desktop\\Dataset", 'word_vecs')) # Necessary!!!
# # Search for first zero
# l = 0
# r = len(x_word2vec)-1
# m = 0
# while l < r:
#     m = (l + m + 1)//2
#     if x_word2vec[m] == 0:
#         r = m
#     else:
#         l = m+1
# print(m)<jupyter_output><empty_output><jupyter_text>Load ML libraries<jupyter_code>import tensorflow as tf
from tensorflow import keras
classes = [b'fake', b'satire', b'bias', b'conspiracy', b'state', b'junksci', b'hate', b'clickbait', b'unreliable', b'political', b'reliable', b'unknown']

# Generate onehot mapping
hot = {}
for idx, clss in enumerate(classes):
    hot[clss] = tf.reshape(tf.constant([0.] * max(idx, 0) + [1.] + [0.] * max(len(classes)-idx-1, 0), dtype=np.float32), shape=(1, 12))

len_data = 9408908
test_size = 0.005
batch_size = 1

test_len = int((len_data * test_size)//batch_size) * batch_size
train_len = len_data - test_len
len(hot[b'bias'])
dataset = tf.data.experimental.make_csv_dataset(os.path.join(basepath, 'news.csv'), batch_size=batch_size, select_columns=['content', 'type'], label_name='type', shuffle=True)
for k in dataset:
    a = k
    break
from collections import OrderedDict
k: OrderedDict = k<jupyter_output><empty_output><jupyter_text>def ragged_concat(arr, axis=1):    return tf.RaggedTensor.from_row_lengths(tf.concat(arr, axis=axis), row_lengths=tuple(len(i) for i in arr))conv3 = keras.layers.Conv1D(8, kernel_size=3, padding='same')conv2 = keras.layers.Conv1D(4, kernel_size=2, padding='same')conv5 = keras.layers.Conv1D(4, kernel_size=4, padding='same') l = [conv5(tf.zeros((1, 12, 300))), conv3(tf.zeros((1, 12, 300))), conv2(tf.zeros((1, 12, 300)))] Shape is (Batch, Word Vecs, Word Vec size) 'same' padding fixes the word vec size We have different filter sizes (Last one)tf.concat(l, 2)  Build the model<jupyter_code>class MultiKernel1D(keras.layers.Layer):
    """Supports using multiple kernel sizes for convoltion on text"""
    def __init__(self, filters:tuple=(180, 100), kernels:tuple=(3,5), input_shape=(None, 300)):
        super(MultiKernel1D, self).__init__()
        self.kernels = [keras.layers.Conv1D(filter, kernel, input_shape=input_shape, padding='same') for (filter, kernel) in zip(filters, kernels)]

    def call(self, inputs, training=None):
        # Call each of the kernels
        # Output should be like
        # Input shape is a tensor of size (A, 300)
        # Which means a variable vector of 300-vectors
        # Which represent word embeddings

        # Each convolution outputs a vector that is (A, filters)
        # We need to combine them into a ragged tensor
        # [
        #   C1: [F1: [?, ?, ...], F2: [?, ?, ...], ...]
        #   C2: [F1: [?, ?, ...], F2: [?, ?, ...], ...]
        #   C3: [F1: [?, ?, ...], F2: [?, ?, ...], ...]
        # ]
        # 3 dimensions, ragged middle vector
        return tf.concat([kernel(inputs) for kernel in self.kernels], -1)

        #     Conv 2
        #  /        /F1
        # Conv 1     F2
        # 0 | F1 F2 ... F3
        # 1 | F1 F2  F4 
        # 2 | F1 F2  /
        # 3 | F1 F2 /
        # 4 | F1 F2
        # So 3-d Tensor, or 4-d with batches


        

class NewsNet(keras.Model):
    def __init__(self):
        super(NewsNet, self).__init__()
        self.kernels = MultiKernel1D()
        self.sequential = keras.layers.LSTM(30)
        self.predictor = keras.layers.Dense(len(classes), activation=keras.activations.softmax) # [Real, Fake]
    
    def call(self, x, training=None):

        x = self.kernels(x)
        x = self.sequential(x)
        return self.predictor(x)

newsNet = NewsNet()
optimizer = keras.optimizers.Adam(learning_rate=5e-5)
loss_fn = keras.losses.CategoricalCrossentropy()<jupyter_output><empty_output><jupyter_text>Training the model<jupyter_code>## Note: Rerunning this cell uses the same model variables
model = newsNet

def process(batch):
    #     OrderedDict(
    #   [('content', <tf.Tensor: shape=(1,), dtype=string, numpy=array([b'TExt'], dtype=object)>)],
    #   [('content', <tf.Tensor: shape=(1,), dtype=string, numpy=array([b'TExt'], dtype=object)>)])
    return tf.constant([[[w.vector for w in text_to_nlp(str(text)) if not (w.is_stop or w.lemma_ =='-PRON-' or w.is_punct)] for text in batch['content'].numpy()][0]], dtype=np.float32)
<EMAIL>(input_signature=(tf.TensorSpec(shape=[1, None, 300], dtype=np.float32), tf.TensorSpec(shape=(1, len(classes)), dtype=np.float32)))
def train_step(x, y):
    with tf.GradientTape() as tape:
        logits = model(x, training=True)
        loss_value = loss_fn(y, logits)
    grads = tape.gradient(loss_value, model.trainable_weights)
    optimizer.apply_gradients(zip(grads, model.trainable_weights))
    train_acc_metric.update_state(y, logits)
    return loss_value


train_acc_metric = keras.metrics.CategoricalAccuracy()
val_acc_metric = keras.metrics.CategoricalAccuracy()
<EMAIL>(input_signature=(tf.TensorSpec(shape=[1, None, 300], dtype=np.float32), tf.TensorSpec(shape=(1, len(classes)), dtype=np.float32)))
def test_step(x, y):
    val_logits = model(x, training=False)
    val_acc_metric.update_state(y, val_logits)


# Keep results for plotting
train_loss_results = []
train_accuracy_results = []

import time
epochs = 3
for epoch in range(epochs):
    print("\nStart of epoch %d" % (epoch,))
    start_time = time.time()
    # Iterate over the batches of the dataset.
    step = 0
    for batch_data, label in dataset:
        if (label.numpy()[0] not in hot):
            step += 1
            continue
        if step < train_len:
            loss_value = train_step(process(batch_data), hot[label.numpy()[0]])

            # Log every 50 batches.
            if step % 50 == 0:
                print(
                    "Training loss (for one batch) at step %d: %.4f"
                    % (step, float(loss_value)))
        else:
            if step == train_len:
                # Display metrics at the end of each epoch.
                train_acc = train_acc_metric.result()
                print("Training acc over epoch: %.4f" % (float(train_acc),))

                # Reset training metrics at the end of each epoch
                train_acc_metric.reset_states()
            test_step(process(batch_data), hot[label.numpy()[0]])
        step += 1

    val_acc = val_acc_metric.result()
    val_acc_metric.reset_states()
    print("Validation acc: %.4f" % (float(val_acc),))
    print("Time taken: %.2fs" % (time.time() - start_time))
    step=0


idx = np.random.randint(0, len(true_data))

res = model(tf.constant([[x.vector for x in text_to_nlp(true_data.at[idx, 'text'])]]))
print(res[0])
# Save the model
model.save('news_classifier') 
step = 0
for text, fake in tqdm.tqdm(zip(x_test, y_test)):
    x_batch_val = tf.constant([[x.vector for x in text_to_nlp(text)]])
    y_batch_val = np.reshape([fake], newshape=(1,1))
    test_step(x_batch_val, y_batch_val)
    step += 1
    if step % 100 == 0:
        print_remove("Validation acc: %.4f" % (float(val_acc_metric.result()),))
val_acc = val_acc_metric.result()
print_remove("Validation acc: %.4f" % (float(val_acc),))
print_remove("Time taken: %.2fs" % (time.time() - start_time))
val_acc_metric.reset_states()<jupyter_output><empty_output><jupyter_text>Exercise 4<jupyter_code># train_y_pred = model.predict([[x.vector for x in text_to_nlp(y)] for y in x_test])
# print('Train accuracy', accuracy_score(y_test, train_y_pred))

for text, fake in tqdm.tqdm(zip(x_test, y_test)):
    x_batch_val = tf.constant([[x.vector for x in text_to_nlp(text)]])
    y_batch_val = np.reshape([fake], newshape=(1,1))
    test_step(x_batch_val, y_batch_val)

val_acc = val_acc_metric.result()
val_acc_metric.reset_states()
print_remove("Validation acc: %.4f" % (float(val_acc),))
print_remove("Time taken: %.2fs" % (time.time() - start_time))<jupyter_output><empty_output><jupyter_text>We can see that we are not doing very well, but we are doing better than 50%. We can do the same for the val data to see how we are doing on unseen data, which is more valuable for us if we want to make predictions on new websites. Fill in the code below to evaluate val accuracy! We appear to be doing a bit worse on the val data, not much better than chance. To better understand the performance of our binary classification model, we should seek to better understand the mistakes that it is making. Specifically, when our model makes a mistake (about 40% of the time), are these mistakes false negatives or false positives?  Confusion Matrix  To answer these questions, we produce and analyze the confusion matrix. The confusion matrix is a matrix that shows the following:![Confusion Matrix](https://cdn-images-1.medium.com/max/1600/1*Z54JgbS4DUwWSknhDCvNTQ.png)where the terms mean* TP (True Positive) = You predicted positive (fake in our case, since fake has a label of 1) and it’s true.* FP (False Positive) = You predicted positive and it’s false.* FN (False Negative) = You predicted negative and it’s false.* TN (True Negative) = You predicted negative and it’s true.Common MetricsFrom the confusion matrix, we can extract commonly used metrics like precision (TP/(TP + FP)) and recall (TP/(TP + FN)). * Precision quantifies how often the things we classify as positive are actually positive. For our task, this measures what fraction of the sites we classify as fake are actually fake. * Recall quantifies what fraction of actually positive examples we classify as positive. In our case, this is the fraction of fake news websites that we actually identify as fake.Finally, a useful score to summarize both precision and recall is the F-1 score. This is just a simple function (the harmonic mean) of precision and recall, shown in the summary below: Exercise 5 |  Using the Confusion Matrix  Run the cell below to create the confusion matrix for our own model.<jupyter_code>print('Confusion matrix:')
print(confusion_matrix(y_train, train_y_pred))<jupyter_output><empty_output><jupyter_text>A Confusion Matrix can quickly tell you how well your model is doing. The primary way to figure this out is to calculate the Error Rate. The Error Rate is:   (FP) + (FN)) / (TP + FP + FN + TN).This is just all the false predictions (False Negative + False Positive) divided by all the predictions added together.  Use the Confusion Matrix we just created to calculate the Error Rate for our model.<jupyter_code>### YOUR CODE HERE ###

### END CODE HERE ###<jupyter_output><empty_output><jupyter_text>Exercise 6<jupyter_code>print(val_y_pred)<jupyter_output><empty_output><jupyter_text>We can see that we have many false negatives, and not as many false positives. Why is this the case? If we print out *val_y_pred*, we can see that our model is mostly predicting 0's (websites are real). What fraction of predictions in *val_y_pred* are 1's? Hint: you may find *np.mean* useful.<jupyter_code>### YOUR CODE HERE ###

### END CODE HERE ###<jupyter_output><empty_output><jupyter_text>Why so many 0's? The only information we are giving our model is its domain name extension. It's natural that the model would learn that websites with ".biz" extensions are unlikely to be reliable news websites, but it is still the case that most websites in the dataset (fake and real) have ".com" extensions. Thus, our model will misclassify many fake news websites with ".com" extensions as real.<jupyter_code>prf = precision_recall_fscore_support(val_y, val_y_pred)

print('Precision:', prf[0][1])
print('Recall:', prf[1][1])
print('F-Score:', prf[2][1])<jupyter_output><empty_output><jupyter_text>Again, the precision and recall metrics suggest that when we classify a website as fake, we are usually right, but we are not doing great at classifying these websites as fake frequently enough. Using Keywords for a Stronger Baseline  The key problem with our model in its current state is that it simply does not have enough information. This should not be a surprise–it was pretty unlikely in the first place that domain name extensions would be enough. If you like, feel free to add a few more extensions in the “featurizer” above and re-run all the code for evaluation–you'll find it doesn't make much of a difference.Where can we get more information about webpages? From the HTML! Remember that the HTML contains all of the text and structure of a webpage. If we cleverly choose features from the HTML to feed into our logistic regression model, we will drastically improve our performance. We saw yesterday that probing hypotheses related to the counts of hypotheses words produced interesting results, and we will continue in this direction today to produce a model that leverages these differences in word frequencies.  Exercise 7: Instructor-Led Discussion on Better Input Features The below code introduces a better featurizer that counts the number of keywords (normalized using the *log* function) in the HTML. Normalizing the counts is a trick that prevents the featurized values from becoming too extreme. Read the code and make sure you understand what it is doing. Then add "sports" and "finance" as additional keywords to expand our model.**Run the below code and discuss what it is doing as a class. Add in additional keywords to further expand our model as you see fit.** Exercise 8 Let's run and evaluate the above featurizer. Add in code to fit the model, compute train accuracy, val accuracy, val confusion matrix, and val precision, recall, and F1-Score, just as before.<jupyter_code>train_X, train_y, feature_descriptions = prepare_data(train_data, keyword_featurizer)
val_X, val_y, feature_descriptions = prepare_data(val_data, keyword_featurizer)

print('Number of features per example:', len(train_X[0]))
print('Feature descriptions:')
print(feature_descriptions)
print()
  
baseline_model = LogisticRegression()

### YOUR CODE HERE ###

### END CODE HERE ###<jupyter_output><empty_output><jupyter_text>Interpreting our Model  Instructor-Led Discussion: Interpreting Input Variables As mentioned earlier, a key motivation for using a simpler model is interpretability.We've learned that the prediction of a logistic regression classifier is just the output of a multiplication with model weights, followed by a non-linear transformation (sigmoid). Because the sigmoid function is always increasing (monotonic) on its domain (see below), we know that if the dot product (or multiplication of vectors) between model weights and input features is large, then the output prediction will be closer to 1. If the dot product is small, then the output prediction will be closer to 0.![Sigmoid](https://cdn-images-1.medium.com/max/2400/1*RqXFpiNGwdiKBWyLJc_E7g.png)Thus, the weights corresponding to features tell us whether the features are important in the classification. If the weight corresponding to the feature ".net domain" has a large positive value, then websites with ".net" domains are more likely to be classified as fake (since fake has label 1). If it has a large negative value, then these websites are more likely to be classified as real. If it has value close to 0, then the feature may not be useful (at least, it may not be useful given that the other features are present). Using Feature Descriptions Let's see what weights our model learned. The code below uses *feature_descriptions* and the weights, or coefficients, of the model and sorts them in ascending order.<jupyter_code>sorted(zip(feature_descriptions, baseline_model.coef_[0].tolist()), key=lambda x: x[1])<jupyter_output><empty_output><jupyter_text>Exercise 9Answer the following questions:* What features have positive weight (most predictive of being fake)? What does that indicate?* Which ones have negative weight (most predictive of being real)? What does that indicate?* Which ones have close to 0 weight? * Are there any feature weights that surprise you? * Try coming up with explanations for why the feature weights are the way they are. Does this help you come up with new feature ideas? (~15 minutes)<jupyter_code>'''
- 
'''<jupyter_output><empty_output><jupyter_text>Instructor-Led Discussion: Final Interpretation of Inputs Exercise 10 |  Final BaselineFinally, play around with the last few cells, adding more keywords and domain names to see how the results change. Note that "keywords" can be a variety of things: English words, English phrases (spaces are allowed), HTML tags, and any other string present in HTML. Also notice how the weights on different features vary–you may observe some interesting effects. When you are done, run the cell below to run evaluations again!<jupyter_code>train_y_pred = baseline_model.predict(train_X)
print('Train accuracy', accuracy_score(train_y, train_y_pred))

val_y_pred = baseline_model.predict(val_X)
print('Val accuracy', accuracy_score(val_y, val_y_pred))

print('Confusion matrix:')
print(confusion_matrix(val_y, val_y_pred))

prf = precision_recall_fscore_support(val_y, val_y_pred)

print('Precision:', prf[0][1])
print('Recall:', prf[1][1])
print('F-Score:', prf[2][1])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11133772133123204, 'fraction_numerical': 0.01205154352461296, 'mean_word_length': 4.473110096397767, 'pattern_counts': {'":': 0, '<': 74, '<?xml version=': 0, '>': 72, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '519429', 'license': 'MIT', 'n_tokens_mistral': 6988, 'n_tokens_neox': 6687, 'n_words': 2278, 'path': 'Notebook.ipynb', 'repo_name': 'master-puller/Anti-Fake-News-AI', 'size': '21586'}