<jupyter_start><jupyter_text>Sarcasm Detection using Convolutional Neural Networks@author: <NAME>  Importing the libraries<jupyter_code>import pandas as pd
import numpy as np
import re
import json
import gensim
import math
import nltk
nltk.download('stopwords')
nltk.download('wordnet')
from nltk.corpus import stopwords
from nltk.stem.porter import PorterStemmer 
from nltk.stem.snowball import SnowballStemmer
from nltk.stem.wordnet import WordNetLemmatizer
from gensim.models import KeyedVectors
import keras 
from keras.models import Sequential, Model 
from keras import layers
from keras.layers import Dense, Dropout, Conv1D, GlobalMaxPooling1D
import h5py<jupyter_output>[nltk_data] Downloading package stopwords to
[nltk_data]     /Users/amankedia/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package wordnet to
[nltk_data]     /Users/amankedia/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!
Using TensorFlow backend.<jupyter_text>Reading data<jupyter_code>def parse_data(file):
    for l in open(file,'r'):
        yield json.loads(l)
data = list(parse_data('Dataset/Sarcasm_Headlines_Dataset_v2.json'))
df = pd.DataFrame(data)<jupyter_output><empty_output><jupyter_text>Basic Data Understanding<jupyter_code>df.head(5)<jupyter_output><empty_output><jupyter_text>Sarcastic Headline<jupyter_code>df['headline'][4]<jupyter_output><empty_output><jupyter_text>Non-sarcastic Headline<jupyter_code>df['headline'][1]
df.pop('article_link')
df.head(5)
len(df)
classes = np.unique(np.array(df['is_sarcastic']))
classes<jupyter_output><empty_output><jupyter_text>Data preprocessing<jupyter_code>def text_clean(corpus):
    '''
    Purpose : Function to keep only alphabets, digits and certain words (punctuations, qmarks, tabs etc. removed)
    
    Input : Takes a text corpus, 'corpus' to be cleaned along with a list of words, 'keep_list', which have to be retained
            even after the cleaning process
    
    Output : Returns the cleaned text corpus
    
    '''
    cleaned_corpus = pd.Series()
    for row in corpus:
        qs = []
        for word in row.split():
            p1 = re.sub(pattern='[^a-zA-Z0-9]',repl=' ',string=word)
            p1 = p1.lower()
            qs.append(p1)
        cleaned_corpus = cleaned_corpus.append(pd.Series(' '.join(qs)))
    return cleaned_corpus
def stopwords_removal(corpus):
    stop = set(stopwords.words('english'))
    corpus = [[x for x in x.split() if x not in stop] for x in corpus]
    return corpus
def lemmatize(corpus):
    lem = WordNetLemmatizer()
    corpus = [[lem.lemmatize(x, pos = 'v') for x in x] for x in corpus]
    return corpus
def stem(corpus, stem_type = None):
    if stem_type == 'snowball':
        stemmer = SnowballStemmer(language = 'english')
        corpus = [[stemmer.stem(x) for x in x] for x in corpus]
    else :
        stemmer = PorterStemmer()
        corpus = [[stemmer.stem(x) for x in x] for x in corpus]
    return corpus
def preprocess(corpus, cleaning = True, stemming = False, stem_type = None, lemmatization = False, remove_stopwords = True):
    
    '''
    Purpose : Function to perform all pre-processing tasks (cleaning, stemming, lemmatization, stopwords removal etc.)
    
    Input : 
    'corpus' - Text corpus on which pre-processing tasks will be performed
    
    'cleaning', 'stemming', 'lemmatization', 'remove_stopwords' - Boolean variables indicating whether a particular task should 
                                                                  be performed or not
    'stem_type' - Choose between Porter stemmer or Snowball(Porter2) stemmer. Default is "None", which corresponds to Porter
                  Stemmer. 'snowball' corresponds to Snowball Stemmer
    
    Note : Either stemming or lemmatization should be used. There's no benefit of using both of them together
    
    Output : Returns the processed text corpus
    
    '''
    if cleaning == True:
        corpus = text_clean(corpus)
    
    if remove_stopwords == True:
        corpus = stopwords_removal(corpus)
    else :
        corpus = [[x for x in x.split()] for x in corpus]
    
    if lemmatization == True:
        corpus = lemmatize(corpus)
        
        
    if stemming == True:
        corpus = stem(corpus, stem_type)
    
    corpus = [' '.join(x) for x in corpus]
        

    return corpus
df['headline']
headlines = preprocess(df['headline'], lemmatization = True, remove_stopwords = True)
headlines[0:5]<jupyter_output><empty_output><jupyter_text>Loading Word2Vec Model<jupyter_code>model = KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin', binary=True)<jupyter_output><empty_output><jupyter_text>Defining model parameters<jupyter_code>MAX_LENGTH = 10
VECTOR_SIZE = 300<jupyter_output><empty_output><jupyter_text>Data Vectorization and Standardization<jupyter_code>def vectorize_data(data):
    
    vectors = []
    
    padding_vector = [0.0] * VECTOR_SIZE
    
    for i, data_point in enumerate(data):
        data_point_vectors = []
        count = 0
        
        tokens = data_point.split()
        
        for token in tokens:
            if count >= MAX_LENGTH:
                break
            if token in model.wv.vocab:
                data_point_vectors.append(model.wv[token])
            count = count + 1
        
        if len(data_point_vectors) < MAX_LENGTH:
            to_fill = MAX_LENGTH - len(data_point_vectors)
            for _ in range(to_fill):
                data_point_vectors.append(padding_vector)
        
        vectors.append(data_point_vectors)
        
    return vectors
vectorized_headlines = vectorize_data(headlines)<jupyter_output>/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/ipykernel_launcher.py:16: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).
  app.launch_new_instance()
/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/ipykernel_launcher.py:17: DeprecationWarning: Call to deprecated `wv` (Attribute will be removed in 4.0.0, use self instead).<jupyter_text>Data Validation<jupyter_code>for i, vec in enumerate(vectorized_headlines):
    if len(vec) != MAX_LENGTH:
        print(i)
len(vectorized_headlines[1])
len(vectorized_headlines)<jupyter_output><empty_output><jupyter_text>Train Test Split and Conversion of Data Into Form expected by Convolutional Neural Network<jupyter_code>train_div = math.floor(0.7 * len(vectorized_headlines))
train_div
X_train = vectorized_headlines[:train_div]
y_train = df['is_sarcastic'][:train_div]
X_test = vectorized_headlines[train_div:]
y_test = df['is_sarcastic'][train_div:]

print('The size of X_train is:', len(X_train), '\nThe size of y_train is:', len(y_train),
      '\nThe size of X_test is:', len(X_test), '\nThe size of y_test is:', len(y_test))
X_train = np.reshape(X_train, (len(X_train), MAX_LENGTH, VECTOR_SIZE))
X_test = np.reshape(X_test, (len(X_test), MAX_LENGTH, VECTOR_SIZE))
y_train = np.array(y_train)
y_test = np.array(y_test)<jupyter_output><empty_output><jupyter_text>Defining Neural Network Model Parameters<jupyter_code>FILTERS=8
KERNEL_SIZE=3
HIDDEN_LAYER_1_NODES=10
HIDDEN_LAYER_2_NODES=5
DROPOUT_PROB=0.35
NUM_EPOCHS=10
BATCH_SIZE=50<jupyter_output><empty_output><jupyter_text>Defining our CNN+FeedForward Neural Network for Detecting Sarcasm<jupyter_code>model = Sequential()

model.add(Conv1D(FILTERS,
                 KERNEL_SIZE,
                 padding='same',
                 strides=1,
                 activation='relu', 
                 input_shape = (MAX_LENGTH, VECTOR_SIZE)))
model.add(GlobalMaxPooling1D())
model.add(Dense(HIDDEN_LAYER_1_NODES, activation='relu'))
model.add(Dropout(DROPOUT_PROB))
model.add(Dense(HIDDEN_LAYER_2_NODES, activation='relu'))
model.add(Dropout(DROPOUT_PROB))
model.add(Dense(1, activation='sigmoid'))
print(model.summary())<jupyter_output>Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv1d_1 (Conv1D)            (None, 10, 8)             7208      
_________________________________________________________________
global_max_pooling1d_1 (Glob (None, 8)                 0         
_________________________________________________________________
dense_1 (Dense)              (None, 10)                90        
_________________________________________________________________
dropout_1 (Dropout)          (None, 10)                0         
_________________________________________________________________
dense_2 (Dense)              (None, 5)                 55        
_________________________________________________________________
dropout_2 (Dropout)          (None, 5)                 0         
______________________________________________________[...]<jupyter_text>Model building and training<jupyter_code>model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
training_history = model.fit(X_train, y_train, epochs=NUM_EPOCHS, batch_size=BATCH_SIZE)<jupyter_output>Epoch 1/10
20033/20033 [==============================] - 2s 88us/step - loss: 0.6523 - acc: 0.6119
Epoch 2/10
20033/20033 [==============================] - 1s 73us/step - loss: 0.5698 - acc: 0.7280
Epoch 3/10
20033/20033 [==============================] - 1s 69us/step - loss: 0.5224 - acc: 0.7642
Epoch 4/10
20033/20033 [==============================] - 1s 73us/step - loss: 0.4946 - acc: 0.7842
Epoch 5/10
20033/20033 [==============================] - 1s 74us/step - loss: 0.4703 - acc: 0.8002
Epoch 6/10
20033/20033 [==============================] - 2s 79us/step - loss: 0.4460 - acc: 0.8124
Epoch 7/10
20033/20033 [==============================] - 2s 75us/step - loss: 0.4302 - acc: 0.8251
Epoch 8/10
20033/20033 [==============================] - 1s 70us/step - loss: 0.4128 - acc: 0.8326
Epoch 9/10
20033/20033 [==============================] - 1s 71us/step - loss: 0.3986 - acc: 0.8410
Epoch 10/10
20033/20033 [==============================] - 1s 71us/step - loss: 0.3850 - acc: 0.8488[...]<jupyter_text>Model Evaluation<jupyter_code>loss, accuracy = model.evaluate(X_test, y_test, verbose=False)
print("Testing Accuracy:  {:.4f}".format(accuracy))
model_structure = model.to_json()
with open("Output Files/sarcasm_detection_model_cnn.json", "w") as json_file:
    json_file.write(model_structure)
model.save_weights("Output Files/sarcasm_detection_model_cnn.h5")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1339378483045244, 'fraction_numerical': 0.03457069991499008, 'mean_word_length': 3.856880733944954, 'pattern_counts': {'":': 0, '<': 59, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '387811', 'license': 'MIT', 'n_tokens_mistral': 3725, 'n_tokens_neox': 3392, 'n_words': 761, 'path': 'Chapter09/Sarcasm Detection using CNNs.ipynb', 'repo_name': 'arifmudi/Hands-On-Python-Natural-Language-Processing', 'size': '10591'}