<jupyter_start><jupyter_text>iMDb Movie Review Sentiment Analysis  Importing Libraries<jupyter_code>import urllib
import os
import tarfile
import re
import numpy as np
import matplotlib as mpl
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Loading Data<jupyter_code>DOWNLOAD_PATH = os.path.abspath("C:\\Storage\\JupyterLab\\Projects\\Machine Learning\\imdb_review_classificaton\\data")
DOWNLOAD_URL = "http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz"
"""
Download the data from the given source URL
and extract it.
"""

def download_data(download_path = DOWNLOAD_PATH, download_url = DOWNLOAD_URL):
    
    # Creating the directory if it does not exist
    os.makedirs(download_path, exist_ok=True)
    
    # Creating the path to download the data
    imdb_path = os.path.join(download_path, "imdb_data.tar.gz")
    
    # Downloading data
    urllib.request.urlretrieve(download_url, imdb_path)
    
    # Opening the tar file
    imdb_tar = tarfile.open(imdb_path)
    
    # Extracting the tar file
    imdb_tar.extractall(path=download_path)
    
    # Closing the connection to the file
    imdb_tar.close()
# Downloading data and extracting.
download_data()
# Training set path
train_neg_path = os.path.abspath('../data/aclImdb/train/neg/')
train_pos_path = os.path.abspath('../data/aclImdb/train/pos/')

# Test set path
test_neg_path = os.path.abspath('../data/aclImdb/test/neg/')
test_pos_path = os.path.abspath('../data/aclImdb/test/pos/')
"""
Read all the files from the downloaded data
and label them accordingly.
"""

def read_data_from_files(train_neg_path = train_neg_path, train_pos_path = train_pos_path,
    test_neg_path = test_neg_path, test_pos_path = test_pos_path):
    
    # Training and testing set
    test_data = []
    train_data = []
    
    # Building train set
    for path, label in ((train_neg_path, 0), (train_pos_path, 1)):
        
        # List all files in the given path
        for filepath in os.listdir(path):
            
            # Open the file present
            with open(os.path.join(path, filepath), "rb") as file:
                
                # Append the file along with the label
                train_data.append([file.read(), label])
    
    # Building the test set
    for path, label in ((test_neg_path, 0), (test_pos_path, 1)):
        
        # List all files in the given path
        for filepath in os.listdir(path):
            
            # Open the file present
            with open(os.path.join(path, filepath), "rb") as file:
                
                # Append the file along with the label
                test_data.append([file.read(), label])
                
    # Return both train and test set
    return train_data, test_data
# Generating Test and Train set
train_, test_ = read_data_from_files()
len(train_), len(test_)
train_set = np.array(train_)
test_set = np.array(test_)

train_set.shape, test_set.shape
# Getting thw features and labels segregated
X_train, X_test, y_train, y_test = train_set[:, 0], test_set[:, 0], train_set[:, 1], test_set[:, 1]

y_train = y_train.astype(np.uint8)
y_test = y_test.astype(np.uint8)
X_train.shape, X_test.shape, y_train.shape, y_test.shape
X_train[1]
X_test[-1]
y_train[-1]
y_test[-1]<jupyter_output><empty_output><jupyter_text>Data Cleaning<jupyter_code>"""
Convert all unneccesary characters like special characters,
numbers and html tags to mere spaces.
"""

from sklearn.base import BaseEstimator, TransformerMixin

class ConvertAllToTextTransformer(BaseEstimator, TransformerMixin):
    def __init__(self, remove_html = True, keep_only_alphabets = True, lower_characters = True):
        self.lower_characters = lower_characters
        self.remove_html = remove_html
        self.keep_only_alphabets = keep_only_alphabets
        
    def fit(self, X, y = None):
        return self
    
    def transform(self, X, y = None):
        
        # Final transformed text array.
        transformed = []
        
        # Looping over all reviews.
        for review in X:
            
            # Creating a copy
            text = review
            
            # Case 1: Lowering all characters
            if self.lower_characters:
                text = text.lower()
                
            # Case 2: Remove all possible HTML tags
            if self.remove_html:
                text = re.sub(b"</?[a-z]*\s*/?>", b' ', text, flags= re.IGNORECASE | re.MULTILINE)
                
            # Case 3: Remove all special characters and numbers.
            if self.keep_only_alphabets:
                text = re.sub(b"[^a-z]", b' ', text, flags= re.IGNORECASE)
            
            # Replace all extra spaces to single space
            text = re.sub(b"\s+", b' ', text)
            
            # Append to the transformed array.
            transformed.append(text)
        
        # Return clean data
        return np.array(transformed)
X_few = X_train[:3]
transformer = ConvertAllToTextTransformer()
X_few_transformed = transformer.fit_transform(X_few)
X_few
X_few_transformed
"""
Converting all words to vectors
of counts of words in the sentence
using CountVectorizer
"""

from sklearn.feature_extraction.text import CountVectorizer

#Creating a CountVectorizer object
count_vect = CountVectorizer()

# Transforming data
X_few_counts = count_vect.fit_transform(X_few_transformed)

# Displaying shape
X_few_counts.shape
"""
Converting all occurences to
frequencies with TfidfTransformer
"""

from sklearn.feature_extraction.text import TfidfTransformer

#Creating a TfidfTransformer object
tfidf_ = TfidfTransformer()

# Transforming data
X_few_tfidf= tfidf_.fit_transform(X_few_counts)

# Displaying shape
X_few_tfidf.shape<jupyter_output><empty_output><jupyter_text>Building A Model<jupyter_code>"""
Building a pipeline for data cleaning
and prediction to take place simultaneously
"""

from sklearn.pipeline import Pipeline
from sklearn.naive_bayes import MultinomialNB

# Building a text classification pipeline
text_clf = Pipeline([
    ('convert_all_2_text', ConvertAllToTextTransformer()),
    ('count_vect', CountVectorizer()),
    ('tfidf', TfidfTransformer()),
    ('clf', MultinomialNB())
])
# Training the classifier
text_clf.fit(X_train, y_train)
from sklearn.metrics import accuracy_score

# Getting predicted data
y_pred = text_clf.predict(X_test)

# Accuracy Score
accuracy_score(y_pred=y_pred, y_true=y_test)
from sklearn.metrics import classification_report

print(classification_report(y_pred=y_pred, y_true=y_test))
from sklearn.metrics import confusion_matrix

confusion_matrix(y_pred=y_pred, y_true=y_test)
from sklearn.linear_model import SGDClassifier

sgd_clf = Pipeline([
    ('convert_all_2_text', ConvertAllToTextTransformer()),
    ('count_vect', CountVectorizer()),
    ('tfidf', TfidfTransformer()),
    ('clf', SGDClassifier(loss='hinge',
                          penalty='l2',
                          alpha=1e-3,
                          random_state=42,
                          max_iter=5,
                          tol=None)),
])
# Training the classifier
sgd_clf.fit(X_train, y_train)
from sklearn.metrics import accuracy_score

# Getting predicted data
y_pred = sgd_clf.predict(X_test)

# Accuracy Score
accuracy_score(y_pred=y_pred, y_true=y_test)
from sklearn.ensemble import RandomForestClassifier

for_clf = Pipeline([
    ('convert_all_2_text', ConvertAllToTextTransformer()),
    ('count_vect', CountVectorizer()),
    ('tfidf', TfidfTransformer()),
    ('clf', RandomForestClassifier(n_estimators=500))
])
# Training the classifier
for_clf.fit(X_train, y_train)
from sklearn.metrics import accuracy_score

# Getting predicted data
y_pred = for_clf.predict(X_test)

# Accuracy Score
accuracy_score(y_pred=y_pred, y_true=y_test)
print(classification_report(y_pred=y_pred, y_true=y_test))
confusion_matrix(y_pred=y_pred, y_true=y_test)<jupyter_output><empty_output><jupyter_text>Hyperparameter Tuning<jupyter_code># Creating a parameter search space
params = {
    'clf__n_estimators': [50, 100, 200, 300, 400, 500, 600, 700],
    'clf__max_depth': range(10, 110, 10),
    'clf__min_samples_split': range(1, 11),
    'clf__min_samples_leaf': range(1, 11),
}
from sklearn.model_selection import RandomizedSearchCV

# Getting the best model out of the specified parameter search space
final_clf = RandomizedSearchCV(for_clf, param_distributions=params, scoring='roc_auc', cv=3, n_jobs=-1, verbose=3)

# Training
final_clf.fit(X_train, y_train)
# Getting the best classifier
classifier = final_clf.best_estimator_
# Getting predicted data
y_pred = classifier.predict(X_test)

# Accuracy Score
accuracy_score(y_pred=y_pred, y_true=y_test)
confusion_matrix(y_pred=y_pred, y_true=y_test)
print(classification_report(y_pred=y_pred, y_true=y_test))<jupyter_output>precision    recall  f1-score   support

           0       0.85      0.85      0.85     12500
           1       0.85      0.85      0.85     12500

    accuracy                           0.85     25000
   macro avg       0.85      0.85      0.85     25000
weighted avg       0.85      0.85      0.85     25000<jupyter_text>Persisting the model<jupyter_code>import pickle

classifier_pickle_path = 'random_forest_classifier_20202911.pkl'
classifier_pickle = open(classifier_pickle_path, 'wb')
pickle.dump(classifier, classifier_pickle)

classifier_pickle.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09454429437413592, 'fraction_numerical': 0.015420610443475487, 'mean_word_length': 3.3336405529953916, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '397277', 'license': 'MIT', 'n_tokens_mistral': 3224, 'n_tokens_neox': 3077, 'n_words': 715, 'path': 'notebooks/imdb_classification.ipynb', 'repo_name': 'khatrivarun/imdb_review_classification', 'size': '9403'}