<jupyter_start><jupyter_text>Load the data<jupyter_code>import json

def load_data(path):  # load each comment at once
    with open(path, 'r') as f:
        text = f.read()
    
    return text

# Example of the raw data

data1 = load_data('txt_sentoken/neg/cv001_19502.txt')
print(data1)<jupyter_output>the happy bastard's quick movie review 
damn that y2k bug . 
it's got a head start in this movie starring j<NAME> and another baldwin brother ( william this time ) in a story regarding a crew of a tugboat that comes across a deserted russian tech ship that has a strangeness to it when they kick the power back on . 
little do they know the power within . . . 
going for the gore and bringing on a few action sequences here and there , virus still feels very empty , like a movie going for all flash and no substance . 
we don't know why the crew was really out in the middle of nowhere , we don't know the origin of what took over the ship ( just that a big pink flashy thing hit the mir ) , and , of course , we don't know why donald sutherland is stumbling around drunkenly throughout . 
here , it's just " hey , let's chase these people around with some robots " . 
the acting is below average , even from the likes of curtis . 
you're more likely to get a kick out of her work in hallow[...]<jupyter_text>Cleaning the data<jupyter_code># its alwayls important to preprocess the data before feeding in a neural network, in special text, because usually text is 
# always messy.

import re, string
from nltk.corpus import stopwords

def cleaning_data(text, Vocab):
    
    tokens = text.split()  # split the comment into tokens by white space.
    
    # prepare the regex for char filtering
    re_sub = re.compile('[%s]' % re.escape(string.punctuation))
    # remove punctuation for each word
    tokens = [re_sub.sub('', word) for word in tokens]
    
    # remove words that aren't alphabetic
    tokens = [word for word in tokens if word.isalpha()]
    
    # remove the so called stop words (e.g. to, they, it, this ...)
    stop_words = stopwords.words('english')
    tokens = [word for word in tokens if word not in stop_words]
    
    # remove word there have length less then 2
    tokens = [word for word in tokens if len(word) > 1]
    

    tokens = [word for word in tokens if word in Vocab]
    
    return ' '.join(tokens) # return to str
    
    
import os

def load_directory(directory): 
    text = []
    for texts in os.listdir(directory): # load the files in the directory
          if texts.endswith('.txt'):
                 text.append(load_data(os.path.join(directory,texts)))


    return text

neg = load_directory('txt_sentoken/neg')  # load all neg comments
pos = load_directory('txt_sentoken/pos')  # load all pos comments
print(f'size of the negative comments: {len(neg)}')
print(f'size of the positive comments: {len(pos)}')<jupyter_output>size of the negative comments: 1000
size of the positive comments: 1000<jupyter_text>Creating a vocabulary<jupyter_code># to create the vocabulary, we need to take all the words in the dataset

total_comments = neg + pos
len(total_comments)
from collections import Counter

Vocab = Counter()  # start the counter
for comment in total_comments:
    
    # Cleaning the vocab to decrease the sparse matrix
    
    tokens = comment.split()
    re_sub = re.compile('[%s]' % re.escape(string.punctuation))
    tokens = [re_sub.sub('', word) for word in tokens]
    
    tokens = [word for word in tokens if word.isalpha()]
    
    stop_words = stopwords.words('english')
    tokens = [word for word in tokens if word not in stop_words]
    
    tokens = [word for word in tokens if len(word) > 1]
    
    Vocab.update(tokens)  # update the vocab with non known words
    
print(f'Size of the vocabulary: {len(Vocab)}')
print(Vocab.most_common(50))
# reducing the vocab to words that appears at least 3 times in the vocab

min_occurrence = 3 
Vocab = [v for v, i in Vocab.items() if i >= min_occurrence]
print(f'Size of the final vocabulary: {len(Vocab)}')

def save_file(data, filename):
    
    save_data = '\n'.join(data)
    file = open(filename, 'w')
    file.write(save_data)
    file.close()
    

save_file(Vocab, 'Vocab.txt') # saving the vocab<jupyter_output><empty_output><jupyter_text>Cleaning the input data<jupyter_code># The same way that the vocabulary was cleaning it has to be done with the input data too

import numpy as np

def preprocessing_input(pos,neg, Vocab):
    
    input_pos = []
    input_neg = []
    for comment in pos:
        input_pos.append(cleaning_data(comment, Vocab))

    for comment in neg:
        input_neg.append(cleaning_data(comment, Vocab))


    print(len(input_pos))
    print(len(input_neg))

    return np.concatenate((input_pos, input_neg))  # return the data concatenate and ready to be use 



input_data = preprocessing_input(pos, neg, Vocab)  
pos_label = np.ones((len(pos))) # create an array of 1's with the len of positive comments
neg_label = np.zeros((len(neg))) # create an array of 0's with the len of negative comments

label = np.concatenate((pos_label, neg_label), axis = 0 )
print(label.shape)
print(input_data.shape)
# example of the same data showed in the second cell, but now, after the preprocessing phase
print(input_data[1001])<jupyter_output>happy bastards quick movie review damn bug got head start movie starring jamie lee curtis another baldwin brother william time story regarding crew comes across deserted russian tech ship strangeness kick power back little know power within going gore bringing action sequences virus still feels empty like movie going flash substance dont know crew really middle nowhere dont know origin took ship big pink flashy thing hit mir course dont know donald sutherland stumbling around throughout hey lets chase people around robots acting average even likes curtis youre likely get kick work halloween sutherland wasted baldwin well hes acting like baldwin course real star stan robot design cgi occasional good gore shot like picking someones brain robots body parts really turn heres movie otherwise pretty much sunken ship movie<jupyter_text>Tokenizing the vocabulary<jupyter_code># to transform the text into number vectors we can use the Tokenizer from keras

from keras.preprocessing.text import Tokenizer

tokenizer = Tokenizer()
tokenizer.fit_on_texts(Vocab)
X = tokenizer.texts_to_matrix(input_data, mode = 'freq') # we fit the tokenizer into our input data
X.shape # notice that now we have a matrix NxD , where N its the number of samples and T its the number of word in the vocab
# we use train_test_split to shuffle and split the data into training and test set
from sklearn.model_selection import train_test_split



x_train, x_test, y_train, y_test =  train_test_split(X, label, test_size = 0.2, shuffle = True, random_state = 42)
print(x_train.shape)
print(y_train.shape)
print(x_test.shape)
print(y_test.shape)
# Now we create a Neural Netword to train our model
# in this case we'be using a normal neural netword with only 1 hidden layer and 1 output layer

from keras.layers import Dense, Input
from keras.models import Model


i = Input(shape = x_train[0].shape) # the input model size has to be the same size of the number of vocab words
x = Dense(50, activation = 'relu')(i)

x = Dense(1, activation = 'sigmoid')(x)  # its used sigmoid for binary classification

model = Model(i, x)

model.summary()

model.compile(loss = 'binary_crossentropy',
             metrics = ['accuracy'],
             optimizer = 'adam')
r = model.fit(x_train, y_train, epochs = 20)
loss, acc = model.evaluate(x_test, y_test, verbose = 0) # we evaluate the model (average)
print(acc)
import matplotlib.pyplot as plt

epochs = range(1, len(r.history['loss']) + 1)

loss = plt.plot(epochs,  r.history['loss'], 'bo',   label = 'Training loss')
plt.legend()
plt.title('Training Loss')
plt.xlabel('epochs')
plt.ylabel('loss value')
plt.show()

val = plt.plot(epochs, r.history['accuracy'], label = 'Training Validation')
plt.legend()
plt.title('Training Validation')
plt.xlabel('epochs')
plt.ylabel('Validation value')
plt.show()<jupyter_output><empty_output><jupyter_text>Predict new comments<jupyter_code>def new_predicts(data, Vocab):
    
    clean_data = cleaning_data(data, Vocab)
    encoded_data = tokenizer.texts_to_matrix([clean_data], mode = 'freq')
    predict_value = model.predict(encoded_data)
    if predict_value <= 0.5:
        return f'NEGATIVE COMMENT'
    if predict_value > 0.5:
        return f'POSITIVE COMMENT'
    
sample = 'This was a great movie, some parts were good, but others were fantastic.'
sample_clean = cleaning_data(sample, Vocab)
sample_clean
new_predicts(sample, Vocab)
sample2 = 'It was medium film, could be better'
new_predicts(sample2, Vocab)
sample3 = 'It was a good film, but i expected more'
new_predicts(sample3, Vocab)
sample4 = 'It was medium, but the final part was really good.'
new_predicts(sample4, Vocab)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08363028953229398, 'fraction_numerical': 0.0064587973273942095, 'mean_word_length': 4.07687959299039, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 1, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '447868', 'license': 'MIT', 'n_tokens_mistral': 2833, 'n_tokens_neox': 2721, 'n_words': 1083, 'path': 'Sentimental Analysis/.ipynb_checkpoints/SentimentalAnalysis-BoW-checkpoint.ipynb', 'repo_name': 'rodrigoamorimml/NaturalLanguageProcessing', 'size': '8989'}