<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
asar = "./Animal Rescue incidents attended by LFB from Jan 2009.csv"
asar.encode('utf-8').strip()
record = pd.read_csv(asar, encoding= 'unicode_escape')
print(record.SpecialServiceType.unique())
import numpy as np
import pandas as pd
seed = 69 # set random seed for whole document

# Graph plotting
import matplotlib.pyplot as plt
import seaborn as sns

# Displaying dataframes
from IPython.display import display

# Natural Language Processing Thingamajibs
import re
import string
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords
from gensim.models import Word2Vec, word2vec
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_selection import chi2
import gensim

# Classifiers
from sklearn.naive_bayes import GaussianNB, BernoulliNB, MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.svm import LinearSVC, SVC

# Metrics to score classifiers
from sklearn.metrics import confusion_matrix, classification_report
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, auc, roc_curve, log_loss

# Data splitting, CV
from sklearn.model_selection import train_test_split
from sklearn.model_selection import cross_val_score, KFold, StratifiedKFold

# Lifesaver
import pickle
asar = "./Animal Rescue incidents attended by LFB from Jan 2009.csv"
df = pd.read_csv(asar, encoding= 'unicode_escape')
df.rename(columns={'IncidentNumber':'S_NO',
                  'DateTimeOfCall':'DATE_TIME',
                   'CalYear':'YEAR',
                   'FinYear':'FYEAR',
                   'TypeOfIncident':'TYPE',
                   'PumpCount':'PUMP_COUNT',
                   'PumpHoursTotal':'PUMP_TOTAL',
                   'HourlyNotionalCost(｣)':'HOUR_COST',
                   'IncidentNotionalCost(｣)':'INCI_COST',
                   'FinalDescription':'MAIN_DESCRIPTION',
                   'AnimalGroupParent':'ANIMAL_TYPE',
                   'OriginofCall':'ORIGIN',
                   'PropertyType':'PROPERTY',
                   'PropertyCategory':'PROPERTYCATEGORY',
                   'SpecialServiceTypeCategory':'SERVICE_CATEGORY',
                   'SpecialServiceType':'SERVICE_TYPE',
                   'WardCode':'WARD_CODE',
                   'Ward':'WARD',
                   'BoroughCode':'BOROUGHCODE',
                   'Borough':'BOROUGH',
                   'StnGroundName':'GROUND_NAME',
                   'UPRN':'UPRN',
                   'Street':'Street',
                   'USRN':'USRN',
                   'PostcodeDistrict':'POST_CODE',
                   'Easting_m':'EASTING',
                   'Northing_m':'NORTHING',
                   'Easting_rounded':'EASTING_R',
                   'Northing_rounded':'NORTHING_R',
                   'Latitude':'LAT',
                   'Longitude':'LONGI'
                   
                  },inplace=True)
#since we wanna do complaint classification we just do the row with complaints/description in them
df.dropna(axis=0, subset=['MAIN_DESCRIPTION'], 
          inplace=True)
#setting up dataframe for multiclass classification
# Subsetting dataframe into columns useful for our text multi-classification problem
df_product_and_complaint = df[['SERVICE_TYPE', 'MAIN_DESCRIPTION']]



## Pickling our subsetted dataframe
#with open('df_product_and_complaint.pickle', 'wb') as to_write:
#   pickle.dump(df_product_and_complaint, to_write)
# Loading our pickled subsetted dataframe
with open('df_product_and_complaint.pickle', 'rb') as to_read:
    df_product_and_complaint = pickle.load(to_read)
df_product_and_complaint.info()
#no null values nice
df_product_and_complaint.head(5)
df_product_and_complaint.SERVICE_TYPE.value_counts()<jupyter_output><empty_output><jupyter_text>Label encoding<jupyter_code># Applying encoding to the PRODUCT column
df_product_and_complaint['S_ID'] = df_product_and_complaint['SERVICE_TYPE'].factorize()[0]
#factorize[0] arranges the index of each encoded number accordingly to the 
# index of your categorical variables in the service_type column

# Creates a dataframe of the PRODUCT to their respective PRODUCT_ID
category_id_df = df_product_and_complaint[['SERVICE_TYPE', 'S_ID']].drop_duplicates()

# Dictionaries for future use. Creating our cheatsheets for what each encoded label represents.
category_to_id = dict(category_id_df.values) # Creates a service_type: S_ID key-value pair
id_to_category = dict(category_id_df[['S_ID', 'SERVICE_TYPE']].values)  # Creates a S_ID: SERVICE_TYPE key-value pair
df_product_and_complaint.head(10)
#Now that we have encoded our columns, time to move on to the next step -- cleaning the fricken text data
#But save our dataframe here so we don't run into memory issues later and we can start from a new starting point 
## Pickling reduced dataframe
#with open('df_product_and_complaint.pickle', 'wb') as to_write:
#    pickle.dump(df_product_and_complaint, to_write)
# Loading Pickled DataFrame
with open('df_product_and_complaint.pickle', 'rb') as to_read:
    df_product_and_complaint = pickle.load(to_read)
# Reviewing our Loaded Dataframe
print(df_product_and_complaint.info())
print('--------------------------------------------------------------------------------------')
print(df_product_and_complaint.head().to_string())<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 7307 entries, 0 to 7311
Data columns (total 3 columns):
 #   Column            Non-Null Count  Dtype 
---  ------            --------------  ----- 
 0   SERVICE_TYPE      7307 non-null   object
 1   MAIN_DESCRIPTION  7307 non-null   object
 2   S_ID              7307 non-null   int64 
dtypes: int64(1), object(2)
memory usage: 228.3+ KB
None
--------------------------------------------------------------------------------------
                                           SERVICE_TYPE                           MAIN_DESCRIPTION  S_ID
0  Animal assistance involving livestock - Other action  DOG WITH JAW TRAPPED IN MAGAZINE RACK,B15     0
1  Animal assistance involving livestock - Other action          ASSIST RSPCA WITH FOX TRAPPED,B15     0
2        Animal rescue from below ground - Domestic pet                    DOG CAUGHT IN DRAIN,B15     1
3                Animal rescue from water - Farm animal                  HORSE TRAPPED IN LAKE,J17  [...]<jupyter_text>Text Pre-Processing<jupyter_code># Looking at a sample text
sample_complaint = list(df_product_and_complaint.MAIN_DESCRIPTION[:5])[4]

# Converting to a list for TfidfVectorizer to use
list_sample_complaint = []
list_sample_complaint.append(sample_complaint)
list_sample_complaint
# Observing what words are extracted from a TfidfVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer

tf_idf3 = TfidfVectorizer(stop_words='english')
check3 = tf_idf3.fit_transform(list_sample_complaint)

print(tf_idf3.get_feature_names())<jupyter_output>['b15', 'rabbit', 'sofa', 'trapped']<jupyter_text>Model/classifier selection  train/startified/test splits<jupyter_code># Split the data into X and y data sets
X, y = df_product_and_complaint.MAIN_DESCRIPTION, df_product_and_complaint.SERVICE_TYPE
print('X shape:', X.shape, 'y shape:', y.shape)
# Split the data into X and y data sets
X, y = df_product_and_complaint.MAIN_DESCRIPTION, df_product_and_complaint.SERVICE_TYPE
print('X shape:', X.shape, 'y shape:', y.shape)
from sklearn.model_selection import train_test_split

X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, 
                                                            test_size=0.2,   # 80% train/cv, 20% test
                                                            stratify=y,
                                                            random_state=seed)
print('X_train', X_train_val.shape)
print('y_train', y_train_val.shape)
print('X_test', X_test.shape)
print('y_test', y_test.shape)
# Performing Text Pre-Processing

# Import tfidfVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer

# Text Preprocessing
# The text needs to be transformed to vectors so as the algorithms will be able make predictions. 
# In this case it will be used the Term Frequency – Inverse Document Frequency (TFIDF) weight 
# to evaluate how important A WORD is to A DOCUMENT in a COLLECTION OF DOCUMENTS.

# tfidf1 = 1-gram only. 
tfidf1 = TfidfVectorizer(sublinear_tf=True, # set to true to scale the term frequency in logarithmic scale.
                        min_df=5,
                        stop_words='english')

X_train_val_tfidf1 = tfidf1.fit_transform(X_train_val).toarray()
X_test_tfidf1 = tfidf1.transform(X_test)

# tfidf2 = unigram and bigram
tfidf2 = TfidfVectorizer(sublinear_tf=True, # set to true to scale the term frequency in logarithmic scale.
                        min_df=5, 
                        ngram_range=(1,2), # we consider unigrams and bigrams
                        stop_words='english')
X_train_val_tfidf2 = tfidf2.fit_transform(X_train_val).toarray()
X_test_tfidf2 = tfidf2.transform(X_test)


# # StratifiedKFold -> Split 5
# ## We now want to do stratified kfold to preserve the proportion of the category imbalances 
# # (number is split evenly from all the classes)

kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)<jupyter_output><empty_output><jupyter_text>Baseline Model - Train/Stratified CV with MultinomialNB()<jupyter_code>print('1-gram number of (rows, features):', X_train_val_tfidf1.shape)
def metric_cv_stratified(model, X_train_val, y_train_val, n_splits, name):
    """
    Accepts a Model Object, converted X_train_val and y_train_val, n_splits, name
    and returns a dataframe with various cross-validated metric scores 
    over a stratified n_splits kfold for a multi-class classifier.
    """
    # Start timer
    import timeit
    start = timeit.default_timer()
    
    ### Computations below
    
    # StratifiedKFold
    ## We now want to do stratified kfold to preserve the proportion of the category imbalances 
    # (number is split evenly from all the classes)
    from sklearn.model_selection import StratifiedKFold  # incase user forgest to import
    kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)
    
    # Initializing Metrics
    accuracy = 0.0
    micro_f1 = 0.0
    macro_precision = 0.0
    macro_recall = 0.0
    macro_f1 = 0.0
    weighted_precision = 0.0
    weighted_recall = 0.0
    weighted_f1 = 0.0
    roc_auc = 0.0    #Not considering this score in this case
        
    # Storing metrics
    from sklearn.model_selection import cross_val_score  # incase user forgets to import
    accuracy = np.mean(cross_val_score(model, X_train_val, y_train_val, cv=kf, scoring='accuracy'))
    micro_f1 = np.mean(cross_val_score(model, X_train_val, y_train_val, cv=kf, scoring='f1_micro'))
    macro_precision = np.mean(cross_val_score(model, X_train_val, y_train_val, cv=kf, scoring='precision_macro'))
    macro_recall = np.mean(cross_val_score(model, X_train_val, y_train_val, cv=kf, scoring='recall_macro'))
    macro_f1 = np.mean(cross_val_score(model, X_train_val, y_train_val, cv=kf, scoring='f1_macro'))
    weighted_precision = np.mean(cross_val_score(model, X_train_val, y_train_val, cv=kf, scoring='precision_weighted'))
    weighted_recall = np.mean(cross_val_score(model, X_train_val, y_train_val, cv=kf, scoring='recall_weighted'))
    weighted_f1 = np.mean(cross_val_score(model, X_train_val, y_train_val, cv=kf, scoring='f1_weighted'))
    
    # Stop timer
    stop = timeit.default_timer()
    elapsed_time = stop - start
    
    return pd.DataFrame({'Model'    : [name],
                         'Accuracy' : [accuracy],
                         'Micro F1' : [micro_f1],
                         'Macro Precision': [macro_precision],
                         'Macro Recall'   : [macro_recall],
                         'Macro F1score'  : [macro_f1],
                         'Weighted Precision': [weighted_precision],
                         'Weighted Recall'   : [weighted_recall],
                         'Weighted F1'  : [weighted_f1],
                         'Time taken': [elapsed_time]  # timetaken: to be used for comparison later
                        })
# ## Data Science Story:
# # Testing on MultinomialNB first

# # Initialize Model Object
# mnb = MultinomialNB()

# results_cv_stratified_1gram = metric_cv_stratified(mnb, X_train_val_tfidf1, y_train_val, 5, 'MultinomialNB1')
# results_cv_stratified_2gram = metric_cv_stratified(mnb, X_train_val_tfidf2, y_train_val, 5, 'MultinomialNB2')
results_cv_stratified_1gram
results_cv_stratified_2gram<jupyter_output><empty_output><jupyter_text>1-gram<jupyter_code># ## Testing on all Models using 1-gram 

# # Initialize Model Object
# gnb = GaussianNB()
# mnb = MultinomialNB()
# logit = LogisticRegression(random_state=seed)
# randomforest = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)
# linearsvc = LinearSVC()

# ## We do NOT want these two. They take FOREVER to train AND predict
# # knn = KNeighborsClassifier()  
# # decisiontree = DecisionTreeClassifier(random_state=seed)

# # to concat all models
# results_cv_straitified_1gram = pd.concat([metric_cv_stratified(mnb, X_train_val_tfidf1, y_train_val, 5, 'MultinomialNB1'),
#                                            metric_cv_stratified(gnb, X_train_val_tfidf1, y_train_val, 5, 'GaussianNB1'),
#                                            metric_cv_stratified(logit, X_train_val_tfidf1, y_train_val, 5, 'LogisticRegression1'),
#                                            metric_cv_stratified(randomforest, X_train_val_tfidf1, y_train_val, 5, 'RandomForest1'),
#                                            metric_cv_stratified(linearsvc, X_train_val_tfidf1, y_train_val, 5, 'LinearSVC1')
#                                           ], axis=0).reset_index()
results_cv_straitified_1gram
#with open('results_cv_straitified_1gram_df.pickle', 'wb') as to_write:
#     pickle.dump(results_cv_straitified_1gram, to_write)
with open('results_cv_straitified_1gram_df.pickle', 'rb') as to_read:
    results_cv_straitified_1gram = pickle.load(to_read)
## Testing on all Models using 2-gram 

# # Initialize Model Object
# gnb = GaussianNB()
# mnb = MultinomialNB()
# logit = LogisticRegression(random_state=seed)
# knn = KNeighborsClassifier()
# decisiontree = DecisionTreeClassifier(random_state=seed)
# randomforest = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)
# linearsvc = LinearSVC()

# # # to concat all models
# results_cv_straitified_2gram = pd.concat([metric_cv_stratified(mnb, X_train_val_tfidf2, y_train_val, 5, 'MultinomialNB2'),
#                                            metric_cv_stratified(gnb, X_train_val_tfidf2, y_train_val, 5, 'GaussianNB2'),
#                                            metric_cv_stratified(logit, X_train_val_tfidf2, y_train_val, 5, 'LogisticRegression2'),
#                                            metric_cv_stratified(randomforest, X_train_val_tfidf2, y_train_val, 5, 'RandomForest2'),
#                                            metric_cv_stratified(linearsvc, X_train_val_tfidf2, y_train_val, 5, 'LinearSVC2')
#                                           ], axis=0).reset_index()
results_cv_straitified_2gram
results_cv_straitified_1gram
#with open('results_cv_straitified_2gram_df.pickle', 'wb') as to_write:
#    pickle.dump(results_cv_straitified_2gram, to_write)
with open('results_cv_straitified_2gram_df.pickle', 'rb') as to_read:
    results_cv_straitified_2gram = pickle.load(to_read)<jupyter_output><empty_output><jupyter_text>Using GloVe50d<jupyter_code>#Each complaint is mapped to a feature vector by averaging the word embeddings of all words in the review. 
#These features are then fed into the defined function above for train/cross validation.

# ## Using pre-trained GloVe
# #download from https://nlp.stanford.edu/projects/glove/

# glove_file = glove_dir = 'glove.6B.50d.txt'
# w2v_output_file = 'glove.6B.50d.txt.w2v'

# # The following utility converts file formats
# gensim.scripts.glove2word2vec.glove2word2vec(glove_file, w2v_output_file)

# # Now we can load it!
# glove_model_50d = gensim.models.KeyedVectors.load_word2vec_format(w2v_output_file, binary=False)

# # Pickle glove model so we don't have to do the above steps again and keep the damn glove.6b.50d in our folder
# with open('glove_model_50d.pickle', 'wb') as to_write:
#     pickle.dump(glove_model_50d, to_write)
# Load pickled glove_model
with open('glove_model_50d.pickle', 'rb') as to_read:
    glove_model_50d = pickle.load(to_read)
    
num_features = 50 # depends on the pre-trained model you are loading
def complaint_to_wordlist(review, remove_stopwords=False):
    """
    Convert a complaint to a list of words. Removal of stop words is optional.
    """
    # remove non-letters
    review_text = re.sub("[^a-zA-Z]"," ", review)
    
    # convert to lower case and split at whitespace
    words = review_text.lower().split()
    
    # remove stop words (false by default)
    if remove_stopwords:
        stops = set(stopwords.words("english"))
        words = [w for w in words if not w in stops]

    return words    # list of tokenized and cleaned words
# num_features refer to the dimensionality of the model you are using
# model refers to the trained word2vec/glove model
# words refer to the words in a single document/entry

def make_feature_vec(words, model, num_features):
    """
    Average the word vectors for a set of words
    """
    feature_vec = np.zeros((num_features,),  # creates a zero matrix of (num_features, )
                           dtype="float32")  # pre-initialize (for speed)
    
    # Initialize a counter for the number of words in a complaint
    nwords = 0.
    index2word_set = set(model.index2word)  # words known to the model

    
    # Loop over each word in the comment and, if it is in the model's vocabulary, add its feature vector to the total
    for word in words:   # for each word in the list of words
        if word in index2word_set:   # if each word is found in the words known to the model
            nwords = nwords + 1.     # add 1 to nwords
            feature_vec = np.add(feature_vec, model[word])   
    
    # Divide by the number of words to get the average 
    if nwords > 0:
        feature_vec = np.divide(feature_vec, nwords)
    
    return feature_vec
# complaints refers to the whole corpus you intend to put in. 
# Therefore you need to append all these info from your df into a list first

def get_avg_feature_vecs(complaints, model, num_features):
    """
    Calculate average feature vectors for ALL complaints
    """
    # Initialize a counter for indexing 
    counter = 0
    
    # pre-initialize (for speed)
    complaint_feature_vecs = np.zeros((len(complaints),num_features), dtype='float32')  
    
    for complaint in complaints: # each complaint is made up of tokenized/cleaned/stopwords removed words
        complaint_feature_vecs[counter] = make_feature_vec(complaint, model, num_features)
        counter = counter + 1
    return complaint_feature_vecs
# # Tokenizing and vectorizing our Train_Val Complaints (80%)
# clean_train_val_complaints = []
# for complaint in X_train_val:
#     clean_train_val_complaints.append(complaint_to_wordlist(complaint, True))

# X_train_val_glove_features = get_avg_feature_vecs(clean_train_val_complaints, glove_model_50d, num_features)

# # Tokenizing and vectorizing our Test Complaints (20%)
# clean_test_complaints = []
# for complaint in X_train_val:
#     clean_test_complaints.append(complaint_to_wordlist(complaint, True))

# X_test_glove_features = get_avg_feature_vecs(clean_test_complaints, glove_model_50d, num_features)
# ## Run the X_train_val_word2vec_features into our defined function for scoring 

# # Initialize Model Object
# gnb = GaussianNB()
# mnb = MultinomialNB()
# logit = LogisticRegression(random_state=seed)
# randomforest = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)
# linearsvc = LinearSVC()


# # to concat all models
# results_cv_straitified_glove50d = pd.concat([
# #     metric_cv_stratified(mnb, X_train_val_glove_features, y_train_val, 5, 'MultinomialNB_glove50d'),
#      metric_cv_stratified(gnb, X_train_val_glove_features, y_train_val, 5, 'GaussianNB_glove50d'),
#      metric_cv_stratified(logit, X_train_val_glove_features, y_train_val, 5, 'LogisticRegression_glove50d'),
#      metric_cv_stratified(randomforest, X_train_val_glove_features, y_train_val, 5, 'RandomForest_glove50d'),
#      metric_cv_stratified(linearsvc, X_train_val_glove_features, y_train_val, 5, 'LinearSVC_glove50d')
#     ], axis=0).reset_index()
# # Saving Results into a DF
# with open('results_cv_straitified_glove50d.pickle', 'wb') as to_write:
#     pickle.dump(results_cv_straitified_glove50d, to_write)
# Opening Results
with open('results_cv_straitified_glove50d.pickle', 'rb') as to_read:
    results_cv_straitified_glove50d = pickle.load(to_read)
results_cv_straitified_glove50d<jupyter_output><empty_output><jupyter_text>Using GloVe100d<jupyter_code>#del glove_model_50d, results_cv_straitified_glove50d
# ## Using pre-trained GloVe
# # download from https://nlp.stanford.edu/projects/glove/

# num_features = 100 # depends on the pre-trained model you are loading

# glove_file = glove_dir = 'glove.6B.' + str(num_features) + 'd.txt'
# w2v_output_file = 'glove.6B.' + str(num_features) + 'd.txt.w2v'

# # The following utility converts file formats
# gensim.scripts.glove2word2vec.glove2word2vec(glove_file, w2v_output_file)

# # Now we can load it!
# glove_model_100d = gensim.models.KeyedVectors.load_word2vec_format(w2v_output_file, binary=False)

# # Pickle glove model so we don't have to do the above steps again and keep the damn glove.6b.50d in our folder
# with open('glove_model_' + str(num_features) + 'd.pickle', 'wb') as to_write:
#     pickle.dump(glove_model_100d, to_write)
# Load pickled glove_model
with open('glove_model_100d.pickle', 'rb') as to_read:
    glove_model_100d = pickle.load(to_read)
# # For Train_Val Complaints (80%)
# clean_train_val_complaints = []
# for complaint in X_train_val:
#     clean_train_val_complaints.append(complaint_to_wordlist(complaint, True))

# X_train_val_glove_features = get_avg_feature_vecs(clean_train_val_complaints, glove_model_100d, num_features)

# # For Test Complaints (20%)
# clean_test_complaints = []
# for complaint in X_train_val:
#     clean_test_complaints.append(complaint_to_wordlist(complaint, True))

# X_test_glove_features = get_avg_feature_vecs(clean_test_complaints, glove_model_100d, num_features)
# ## Run the X_train_val_word2vec_features into our defined function for scoring 

# # Initialize Model Object
# gnb = GaussianNB()
# mnb = MultinomialNB()
# logit = LogisticRegression(random_state=seed)
# randomforest = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)
# linearsvc = LinearSVC()

# ## We do NOT want these two. They take FOREVER to train AND predict
# # knn = KNeighborsClassifier()  
# # decisiontree = DecisionTreeClassifier(random_state=seed)

# # to concat all models
# results_cv_straitified_glove100d = pd.concat([
# #     metric_cv_stratified(mnb, X_train_val_glove_features, y_train_val, 5, 'MultinomialNB_glove50d'),
#      metric_cv_stratified(gnb, X_train_val_glove_features, y_train_val, 5, 'GaussianNB_glove100d'),
#      metric_cv_stratified(logit, X_train_val_glove_features, y_train_val, 5, 'LogisticRegression_glove100d'),
#      metric_cv_stratified(randomforest, X_train_val_glove_features, y_train_val, 5, 'RandomForest_glove100d'),
#      metric_cv_stratified(linearsvc, X_train_val_glove_features, y_train_val, 5, 'LinearSVC_glove100d')
#     ], axis=0).reset_index()
# with open('results_cv_straitified_glove100d.pickle', 'wb') as to_write:
#     pickle.dump(results_cv_straitified_glove100d, to_write)
# Opening Results
with open('results_cv_straitified_glove100d.pickle', 'rb') as to_read:
    results_cv_straitified_glove100d = pickle.load(to_read)
results_cv_straitified_glove100d<jupyter_output><empty_output><jupyter_text>Using GloVe200d<jupyter_code>#del glove_model_100d, results_cv_straitified_glove100d
# ## Using pre-trained GloVe
# # download from https://nlp.stanford.edu/projects/glove/

# num_features = 200 # depends on the pre-trained model you are loading

# glove_file = glove_dir = 'glove.6B.' + str(num_features) + 'd.txt'
# w2v_output_file = 'glove.6B.' + str(num_features) + 'd.txt.w2v'

# # The following utility converts file formats
# gensim.scripts.glove2word2vec.glove2word2vec(glove_file, w2v_output_file)

# # Now we can load it!
# glove_model_200d = gensim.models.KeyedVectors.load_word2vec_format(w2v_output_file, binary=False)

# # Pickle glove model so we don't have to do the above steps again and keep the damn glove.6b.50d in our folder
# with open('glove_model_' + str(num_features) + 'd.pickle', 'wb') as to_write:
#     pickle.dump(glove_model_200d, to_write)
with open('glove_model_200d.pickle', 'rb') as to_read:
    glove_model_200d = pickle.load(to_read)
# # For Train_Val Complaints (80%)
# clean_train_val_complaints = []
# for complaint in X_train_val:
#     clean_train_val_complaints.append(complaint_to_wordlist(complaint, True))

# X_train_val_glove_features = get_avg_feature_vecs(clean_train_val_complaints, glove_model_200d, num_features)

# #Already run above
# #For Test Complaints (20%)
# clean_test_complaints = []
# for complaint in X_train_val:
#     clean_test_complaints.append(complaint_to_wordlist(complaint, True))

# X_test_glove_features = get_avg_feature_vecs(clean_test_complaints, glove_model_200d, num_features)
# ## Run the X_train_val_word2vec_features into our defined function for scoring 

# # Initialize Model Object
# gnb = GaussianNB()
# mnb = MultinomialNB()
# logit = LogisticRegression(random_state=seed)
# randomforest = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)
# linearsvc = LinearSVC()

# ## We do NOT want these two. They take FOREVER to train AND predict
# # knn = KNeighborsClassifier()  
# # decisiontree = DecisionTreeClassifier(random_state=seed)

# # to concat all models
# results_cv_straitified_glove200d = pd.concat([
# #     metric_cv_stratified(mnb, X_train_val_glove_features, y_train_val, 5, 'MultinomialNB_glove50d'),
#      metric_cv_stratified(gnb, X_train_val_glove_features, y_train_val, 5, 'GaussianNB_glove200d'),
#      metric_cv_stratified(logit, X_train_val_glove_features, y_train_val, 5, 'LogisticRegression_glove200d'),
#      metric_cv_stratified(randomforest, X_train_val_glove_features, y_train_val, 5, 'RandomForest_glove200d'),
#      metric_cv_stratified(linearsvc, X_train_val_glove_features, y_train_val, 5, 'LinearSVC_glove200d')
#     ], axis=0).reset_index()
# with open('results_cv_straitified_glove200d.pickle', 'wb') as to_write:
#     pickle.dump(results_cv_straitified_glove200d, to_write)
with open('results_cv_straitified_glove200d.pickle', 'rb') as to_read:
    results_cv_straitified_glove200d = pickle.load(to_read)
results_cv_straitified_glove200d<jupyter_output><empty_output><jupyter_text>Using GloVe300d<jupyter_code>#del glove_model_200d, results_cv_straitified_glove200d
# ## Using pre-trained GloVe
# # download from https://nlp.stanford.edu/projects/glove/

# num_features = 300 # depends on the pre-trained model you are loading

# glove_file = glove_dir = 'glove.6B.' + str(num_features) + 'd.txt'
# w2v_output_file = 'glove.6B.' + str(num_features) + 'd.txt.w2v'

# # The following utility converts file formats
# gensim.scripts.glove2word2vec.glove2word2vec(glove_file, w2v_output_file)

# # Now we can load it!
# glove_model_300d = gensim.models.KeyedVectors.load_word2vec_format(w2v_output_file, binary=False)

# # Pickle glove model so we don't have to do the above steps again and keep the damn glove.6b.50d in our folder
# with open('glove_model_' + str(num_features) + 'd.pickle', 'wb') as to_write:
#     pickle.dump(glove_model_300d, to_write)
# # Load pickled glove_model
# with open('glove_model_300d.pickle', 'rb') as to_read:
#     glove_model_300d = pickle.load(to_read)

# # For Train_Val Complaints (80%)
# clean_train_val_complaints = []
# for complaint in X_train_val:
#     clean_train_val_complaints.append(complaint_to_wordlist(complaint, True))

# X_train_val_glove_features = get_avg_feature_vecs(clean_train_val_complaints, glove_model_300d, num_features)

# #Already run above
# # For Test Complaints (20%)
# clean_test_complaints = []
# for complaint in X_train_val:
#     clean_test_complaints.append(complaint_to_wordlist(complaint, True))

# X_test_glove_features = get_avg_feature_vecs(clean_test_complaints, glove_model_300d, num_features)
# ## Run the X_train_val_word2vec_features into our defined function for scoring 

# # Initialize Model Object
# gnb = GaussianNB()
# mnb = MultinomialNB()
# logit = LogisticRegression(random_state=seed)
# randomforest = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)
# linearsvc = LinearSVC()

# ## We do NOT want these two. They take FOREVER to train AND predict
# # knn = KNeighborsClassifier()  
# # decisiontree = DecisionTreeClassifier(random_state=seed)

# # to concat all models
# results_cv_straitified_glove300d= pd.concat([
#     # metric_cv_stratified(mnb, X_train_val_glove_features, y_train_val, 5, 'MultinomialNB_glove50d'),
#      metric_cv_stratified(gnb, X_train_val_glove_features, y_train_val, 5, 'GaussianNB_glove300d'),
#      metric_cv_stratified(logit, X_train_val_glove_features, y_train_val, 5, 'LogisticRegression_glove300d'),
#      metric_cv_stratified(randomforest, X_train_val_glove_features, y_train_val, 5, 'RandomForest_glove300d'),
#      metric_cv_stratified(linearsvc, X_train_val_glove_features, y_train_val, 5, 'LinearSVC_glove300d')
#     ], axis=0).reset_index()

# with open('results_cv_straitified_glove300d.pickle', 'wb') as to_write:
#     pickle.dump(results_cv_straitified_glove300d, to_write)
# Opening Results
with open('results_cv_straitified_glove300d.pickle', 'rb') as to_read:
    results_cv_straitified_glove300d = pickle.load(to_read)
results_cv_straitified_glove300d<jupyter_output><empty_output><jupyter_text>GoogleNews Word2Vec300d<jupyter_code>#del glove_model_300d, results_cv_straitified_glove300d
# ## Using pre-trained GoogleNews Word2Vec
# # download from https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit

# num_features = 300 # depends on the pre-trained model you are loading

# # Path to where the word2vec file lives
# google_vec_file = 'GoogleNews-vectors-negative300.bin'

# # Load it!  This might take a few minutes...
# word2vec_model_300d = gensim.models.KeyedVectors.load_word2vec_format(google_vec_file, binary=True)
# # it is just loading all the different weights (embedding) into python


# # Pickle word2vec 300d model so we don't have to do the above steps again and keep the damn file in our folder
# with open('word2vec_model_' + str(num_features) + 'd.pickle', 'wb') as to_write:
#     pickle.dump(word2vec_model_300d, to_write)
# # Load pickled glove_model
# with open('word2vec_model_300d.pickle', 'rb') as to_read:
#     word2vec_model_300d = pickle.load(to_read)
# # For Train_Val Complaints (80%)
# clean_train_val_complaints = []
# for complaint in X_train_val:
#     clean_train_val_complaints.append(complaint_to_wordlist(complaint, True))

# X_train_val_glove_features = get_avg_feature_vecs(clean_train_val_complaints, word2vec_model_300d, num_features)
# ## Run the X_train_val_word2vec_features into our defined function for scoring 

# # Initialize Model Object
# gnb = GaussianNB()
# mnb = MultinomialNB()
# logit = LogisticRegression(random_state=seed)
# randomforest = RandomForestClassifier(n_estimators=100, max_depth=5, random_state=0)
# linearsvc = LinearSVC()

# ## We do NOT want these two. They take FOREVER to train AND predict
# # knn = KNeighborsClassifier()  
# # decisiontree = DecisionTreeClassifier(random_state=seed)

# # to concat all models
# results_cv_straitified_word2vec300d= pd.concat([
# #     metric_cv_stratified(mnb, X_train_val_glove_features, y_train_val, 5, 'MultinomialNB_glove50d'),
#      metric_cv_stratified(gnb, X_train_val_glove_features, y_train_val, 5, 'GaussianNB_word2vec300d'),
#      metric_cv_stratified(logit, X_train_val_glove_features, y_train_val, 5, 'LogisticRegression_word2vec300d'),
#      metric_cv_stratified(randomforest, X_train_val_glove_features, y_train_val, 5, 'RandomForest_word2vec300d'),
#      metric_cv_stratified(linearsvc, X_train_val_glove_features, y_train_val, 5, 'LinearSVC_word2vec300d')
#     ], axis=0).reset_index()
# with open('results_cv_straitified_word2vec300d.pickle', 'wb') as to_write:
#     pickle.dump(results_cv_straitified_word2vec300d, to_write)
# # Opening Results
# with open('results_cv_straitified_word2vec300d.pickle', 'rb') as to_read:
#     results_cv_straitified_word2vec300d = pickle.load(to_read)
results_cv_straitified_word2vec300d
del word2vec_model_300d, results_cv_straitified_word2vec300d<jupyter_output><empty_output><jupyter_text>Compiling Results<jupyter_code>results_compiled = pd.concat([results_cv_straitified_1gram,
                           results_cv_straitified_glove50d,
                           results_cv_straitified_glove100d,
                           results_cv_straitified_glove200d,
                           results_cv_straitified_glove300d]).reset_index().drop(['level_0','index'],axis=1)
results_compiled.columns

with open('results_compiled.pickle', 'wb') as to_write:
    pickle.dump(results_compiled, to_write)

results_compiled<jupyter_output><empty_output><jupyter_text>Sorting in Descending order by Accuracy<jupyter_code># Sorting results to see which one gives us the best results
results_compiled_sorted_by_accuracy = results_compiled.sort_values(by='Accuracy', ascending=False)

with open('results_compiled_sorted_by_accuracy.pickle', 'wb') as to_write:
    pickle.dump(results_compiled_sorted_by_accuracy, to_write)

results_compiled_sorted_by_accuracy
# Retrieving the Model that provides the highest Accuracy
results_highest_accuracy = results_compiled[results_compiled.Accuracy == results_compiled.Accuracy.max()]

with open('results_highest_accuracy.pickle', 'wb') as to_write:
    pickle.dump(results_highest_accuracy, to_write)

results_highest_accuracy<jupyter_output><empty_output><jupyter_text>Splitting to 80% Train and 20% unseen data<jupyter_code># Split the data into X and y data sets
X, y = df_product_and_complaint.MAIN_DESCRIPTION, df_product_and_complaint.SERVICE_TYPE
print('X shape:', X.shape, 'y shape:', y.shape)
# For text classification, ALWAYS split data first before vectorizing.
# This is because you don't want to cheat by having features (words) from the test data already being inside your train data
from sklearn.model_selection import train_test_split

X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, 
                                                            test_size=0.2,   # 80% train/cv, 20% test
                                                            stratify=y,
                                                            random_state=seed)
print('X_train', X_train_val.shape)
print('y_train', y_train_val.shape)
print('X_test', X_test.shape)
print('y_test', y_test.shape)
# tfidf1 = 1-gram only. 
tfidf1 = TfidfVectorizer(sublinear_tf=True, # set to true to scale the term frequency in logarithmic scale.
                        min_df=5,
                        stop_words='english')
X_train_val_tfidf1 = tfidf1.fit_transform(X_train_val).toarray()
X_test_tfidf1 = tfidf1.transform(X_test)
# Initializing our chosen logreg model
logit = LogisticRegression()

# Fitting our model
logit_finalized = logit.fit(X_train_val_tfidf1, y_train_val)

# Obtaining prediction
y_pred = logit_finalized.predict(X_test_tfidf1)
X, y = df_product_and_complaint.MAIN_DESCRIPTION, df_product_and_complaint.SERVICE_TYPE
X_train_val, X_test, y_train_val, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=seed)
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5, stop_words='english')
fitted_tfidf_to_use = tfidf.fit(X_train_val)
with open('fitted_tfidf_to_use.pickle', 'wb') as to_write:
  pickle.dump(fitted_tfidf_to_use, to_write)
 # Pickle trained Model for use in Flask App
 with open('logit_finalized.pickle', 'wb') as to_write:
     pickle.dump(logit_finalized, to_write)
# Now we have a pickled trained model, we can use this for our flask app!
with open('logit_finalized.pickle', 'rb') as to_read:
    logit_finalized = pickle.load(to_read)
# Create a function to calculate the error metrics, since we'll be doing this several times
from sklearn.metrics import confusion_matrix
import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

def conf_matrix(actual, predicted):
    
    # Creates a dataframe of the PRODUCT to their respective PRODUCT_ID
    category_id_df = df_product_and_complaint[['SERVICE_TYPE', 'S_ID']].drop_duplicates()

    # Dictionaries for future use. Creating our cheatsheets for what each encoded label represents.
    category_to_id = dict(category_id_df.values) # Creates a PRODUCT: PRODUCT_ID key-value pair
    id_to_category = dict(category_id_df[['S_ID', 'SERVICE_TYPE']].values)  # Creates a PRODUCT_ID: PRODUCT key-value pair

    conf_mat = confusion_matrix(y_test, y_pred)
    fig, ax = plt.subplots(figsize=(8,8))
    g= sns.heatmap(conf_mat, 
                   annot=True, annot_kws={"size":10},
                   cmap=plt.cm.Reds, square=True,
                   fmt='d',
                   xticklabels=category_id_df.PRODUCT.values, 
                   yticklabels=category_id_df.PRODUCT.values)
    
#     # Changing the size of the xticks and ytick labels
#     ax.set_yticklabels(g.get_yticklabels(), rotation=90, size=10);
#     ax.set_xticklabels(g.get_xticklabels(), size=10);
    
    # Changing axis orientation & setting titles
    ax.set_xlabel('Prediction', size=15)
    ax.set_ylabel('Actual', rotation=0, labelpad=40,size=15)

#     plt.title("CONFUSION MATRIX - {}\n".format(name), size=16);
    
    bottom, top = ax.get_ylim()
    ax.set_ylim(bottom + 0.5, top - 0.5)
fig, ax = plt.subplots(figsize=(10,8))

ax = sns.countplot(y='SERVICE_TYPE', 
                   data=df_product_and_complaint, 
                   order=df_product_and_complaint['SERVICE_TYPE'].value_counts().index,
                   palette='rocket'        # change the color of this graph LATER
                  )

ax.set_title('NUMBER OF COMPLAINTS IN EACH CATEGORY',size=15)

# Setting labels
# Dealing with y-labels
ax.set_ylabel('Service_Type', rotation=0, labelpad=40, size=14)
              
# Dealing with x-labels
ax.set_xlabel('NUMBER OF COMPLAINTS', size=14)

sns.despine()
plt.savefig('freq_of_removed_classes.png', transparency=True)

# These are the ONLY packages we need to use from this point forward!
import pandas as pd
import numpy as np
import pickle
from sklearn.feature_extraction.text import TfidfVectorizer
import sklearn.linear_model as lm
# Loading our saved models
# Consider training on the whole data instead now?
with open('fitted_tfidf_to_use.pickle', 'rb') as to_read:  # loading the fitted tfidf with our 80% trained data
    fitted_tfidf_to_use = pickle.load(to_read)
    
with open('logit_finalized.pickle', 'rb') as to_read:  # loading our model
    logit_finalized = pickle.load(to_read)
# Using our models

complaint = """
bird stuck in wire """

# After fitting the tfidf vectorizor, then you can do transforms!
new_vectorized_complaint = fitted_tfidf_to_use.transform([complaint])

# Fitting vectorized complaint into model
y_customized_prediction = logit_finalized.predict(new_vectorized_complaint)
y_customized_prediction[0]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09463343914814908, 'fraction_numerical': 0.01899667558177319, 'mean_word_length': 4.157664045378368, 'pattern_counts': {'":': 1, '<': 56, '<?xml version=': 0, '>': 58, 'https://': 5, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 40, 'profanity_word_list.json': 5, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '172494', 'license': 'MIT', 'n_tokens_mistral': 14258, 'n_tokens_neox': 13283, 'n_words': 2965, 'path': 'ASAR.ipynb', 'repo_name': 'siddharthshenoy/ASAR', 'size': '40007'}