<jupyter_start><jupyter_text><jupyter_code>import json

def parse_data(file):
    for l in open(file,'r'):
        yield json.loads(l)

data = list(parse_data('Sarcasm_Headlines_Dataset.json'))
sentences = []
labels = []
urls = []
for item in data:
  sentences.append(item['headline'])
  labels.append(item['is_sarcastic'])
  urls.append(item['article_link'])
sentences
len(sentences)
# splitting corpus into training and validation set
training_size = 20000
training_sentences = sentences[0:training_size]
testing_sentences = sentences[training_size:]
training_labels = labels[0:training_size]
testing_labels = labels[training_size:]
from tensorflow.keras.preprocessing.text import Tokenizer
from tensorflow.keras.preprocessing.sequence import pad_sequences

# let's set up our hyper parameters
vocab_size=1000
embedding_dim=16
max_length = 16
trunc_type = 'post'
padding_type = 'post'
oov_tok = '<OOV>'
training_size=20000


tokenizer = Tokenizer(num_words = vocab_size, oov_token=oov_tok) # for unknown words
tokenizer.fit_on_texts(training_sentences) # will convert sentences into tokens
word_index = tokenizer.word_index # we can view our token this way, this will give us a dictionary. here our dictionary would be huge.
print(word_index,'\n')
# once we have the corpus tokenized, we will encode the list of sentences.
training_sequences = tokenizer.texts_to_sequences(training_sentences)
training_padded = pad_sequences(training_sequences, padding=padding_type, maxlen=max_length, truncating=trunc_type)

testing_sequences = tokenizer.texts_to_sequences(testing_sentences)
testing_padded = pad_sequences(testing_sequences, padding=padding_type, maxlen=max_length, truncating=trunc_type)
import tensorflow as tf
model = tf.keras.Sequential([
                             tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length = max_length),
                             tf.keras.layers.GlobalAveragePooling1D(),
                             tf.keras.layers.Dense(24, activation='relu'),
                             tf.keras.layers.Dense(1, activation='sigmoid')
])

# compiling the model
model.compile(loss='binary_crossentropy',
              optimizer = 'adam',
              metrics=['accuracy'])
model.summary()
import numpy as np
training_padded = np.array(training_padded)
training_labels = np.array(training_labels)
testing_padded = np.array(testing_padded)
testing_labels = np.array(testing_labels)
len(testing_labels)
num_of_epochs = 30
history = model.fit(training_padded, training_labels, epochs=num_of_epochs, validation_data=(testing_padded, testing_labels),
                    verbose=2)
# let's plot our loss vs accuracy curve

import matplotlib.pyplot as plt

def plot_loss_curves(history, string):
  plt.plot(history.history[string])
  plt.plot(history.history['val_'+string])
  plt.xlabel('epochs')
  plt.ylabel(string)
  plt.legend([string, 'val_'+string])
  plt.show()

plot_loss_curves(history, 'loss')
plot_loss_curves(history, 'accuracy')<jupyter_output><empty_output><jupyter_text>**note** * not only the meaning of the words matter, but their sequence too does matter. we'll use RNN to solve this issue  SECOND MODEL WITH Bidirectional LSTM<jupyter_code>model_2 = tf.keras.Sequential([
                               tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length = max_length),
                               tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)),
                               tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),
                               tf.keras.layers.Dense(64, activation='relu'),
                               tf.keras.layers.Dense(1, activation='sigmoid')
])

model_2.compile(loss='binary_crossentropy',
                optimizer='adam',
                metrics=['accuracy'])
history_2 = model_2.fit(training_padded, training_labels, epochs=num_of_epochs, validation_data=(testing_padded, testing_labels))<jupyter_output>Epoch 1/30
625/625 [==============================] - 23s 29ms/step - loss: 0.4390 - accuracy: 0.7839 - val_loss: 0.3816 - val_accuracy: 0.8249
Epoch 2/30
625/625 [==============================] - 16s 26ms/step - loss: 0.3626 - accuracy: 0.8350 - val_loss: 0.3780 - val_accuracy: 0.8287
Epoch 3/30
625/625 [==============================] - 16s 26ms/step - loss: 0.3417 - accuracy: 0.8454 - val_loss: 0.3854 - val_accuracy: 0.8274
Epoch 4/30
625/625 [==============================] - 17s 27ms/step - loss: 0.3232 - accuracy: 0.8554 - val_loss: 0.3712 - val_accuracy: 0.8338
Epoch 5/30
625/625 [==============================] - 16s 26ms/step - loss: 0.3087 - accuracy: 0.8629 - val_loss: 0.3732 - val_accuracy: 0.8372
Epoch 6/30
625/625 [==============================] - 16s 26ms/step - loss: 0.2941 - accuracy: 0.8713 - val_loss: 0.3777 - val_accuracy: 0.8317
Epoch 7/30
625/625 [==============================] - 17s 27ms/step - loss: 0.2837 - accuracy: 0.8755 - val_loss: 0.3812 - val_accuracy:[...]<jupyter_text>* we can see, there is definite case of overfitting. we can reduce the number of epochs in this case and resolve the issue of overfitting.<jupyter_code>plot_loss_curves(history_2, 'loss')
plot_loss_curves(history_2, 'accuracy')<jupyter_output><empty_output><jupyter_text>THIRD MODEL WITH CONVOLUTIONAL LAYER<jupyter_code>model_3 = tf.keras.Sequential([
    tf.keras.layers.Embedding(vocab_size, embedding_dim, input_length=max_length),
    tf.keras.layers.Conv1D(128, 5, activation='relu'),
    tf.keras.layers.GlobalMaxPooling1D(),
    tf.keras.layers.Dense(24, activation='relu'),
    tf.keras.layers.Dense(1, activation='sigmoid')
])
model_3.compile(loss='binary_crossentropy',
                optimizer='adam',
                metrics='accuracy')
history_3 = model_3.fit(training_padded, training_labels, epochs=10, validation_data=(testing_padded, testing_labels))<jupyter_output>Epoch 1/10
625/625 [==============================] - 3s 4ms/step - loss: 0.4653 - accuracy: 0.7688 - val_loss: 0.3994 - val_accuracy: 0.8201
Epoch 2/10
625/625 [==============================] - 2s 4ms/step - loss: 0.3520 - accuracy: 0.8410 - val_loss: 0.3926 - val_accuracy: 0.8196
Epoch 3/10
625/625 [==============================] - 2s 4ms/step - loss: 0.3151 - accuracy: 0.8624 - val_loss: 0.3900 - val_accuracy: 0.8235
Epoch 4/10
625/625 [==============================] - 2s 3ms/step - loss: 0.2805 - accuracy: 0.8809 - val_loss: 0.4086 - val_accuracy: 0.8205
Epoch 5/10
625/625 [==============================] - 2s 3ms/step - loss: 0.2476 - accuracy: 0.8955 - val_loss: 0.4276 - val_accuracy: 0.8241
Epoch 6/10
625/625 [==============================] - 2s 4ms/step - loss: 0.2157 - accuracy: 0.9143 - val_loss: 0.4802 - val_accuracy: 0.8182
Epoch 7/10
625/625 [==============================] - 2s 4ms/step - loss: 0.1847 - accuracy: 0.9254 - val_loss: 0.5358 - val_accuracy: 0.8082
Epoch [...]<jupyter_text>* maybe a less number of epochs here would have prevented the slight overfitting that we are getting for third model.<jupyter_code>plot_loss_curves(history_3, 'loss')
plot_loss_curves(history_3, 'accuracy')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1699438202247191, 'fraction_numerical': 0.0702247191011236, 'mean_word_length': 4.274814814814815, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '28962', 'license': 'MIT', 'n_tokens_mistral': 2744, 'n_tokens_neox': 2356, 'n_words': 502, 'path': 'sarcasm_from_headline.ipynb', 'repo_name': '1901010019/SARCASM_FROM_HEADLINES', 'size': '7120'}