<jupyter_start><jupyter_text>Text Classification using Sci-Kit Learn  Consumer Complaint DatabaseThe csv contains complaints data.gov has received about financial products and services. Details about the dataset -     0	- "consumer"     1	- "finance"     2	- "complaint"     3	- "bank account"     4	- "bank service"     5	- "credit card"     6	- "credit report"     7	- "debt collection"     8	- "money transfer"     9	- "mortgage"     10	- "student loan"     11	- "loan"   data urlhttps://catalog.data.gov/dataset/consumer-complaint-database metadatahttps://catalog.data.gov/harvest/object/f88b9cb6-5465-428b-8aa9-0f971eb5538f  Problem The problem is supervised text classification problem, and our goal is to investigate which supervised machine learning methods are best suited to solve it.Given a new complaint comes in, we want to assign it to one of 12 categories. The classifier makes the assumption that each new complaint is assigned to one and only one category. This is multi-class text classification problem. I can’t wait to see what we can achieve!  Input: Consumer_complaint_narrative Output:Product<jupyter_code># load libraries

import os
import gc
import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
%matplotlib inline

from sklearn.feature_selection import chi2
from sklearn.feature_extraction.text import TfidfVectorizer, TfidfTransformer, CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import MultinomialNB

# helper to load dataframe and display 
# how to call this function
# df, col_types, cat_cols = load_dataframe(f_name)

def load_dataframe(f_name):
    '''
    helper f() to load dataframe
    display basic details
       
    '''
    # load dataframe
    df = pd.read_csv(f_name, sep=',', index_col=None, skiprows=None, chunksize=None, 
                 low_memory=False, encoding='utf-8', keep_default_na=True, 
                 na_values='', na_filter=True, nrows=None, engine='c')
    
    # number of columns
    n = len(df.columns)
    
    # display all columns
    pd.options.display.max_columns = n
    
    # columns
    columns = list(df.columns)
    
    # categorical columns
    cat_cols = df.select_dtypes(include=['object'])
    cat_cols = list(cat_cols)
    
    # column types
    col_types = []
    for i in range(n):
        if df.dtypes[i] not in col_types:
            col_types.append(df.dtypes[i])

    print("Basic dataframe details!!")
    
    print("Dataframe dimensions")
    print("")
    
    print("Shape of dataframe: ", df.shape)
    print("")
    
    print("Number of rows: ", len(df))
    print("")
    
    print("Number of columns: ", len(df.columns))
    print("")
    
    print("Is dataframe empty: ", df.empty)
    print("")
    
    print("Describe")
    print(df.describe())
    print("")
    
    print("Describe (including categoricals)")
    print(df.describe(include='all'))
    print("")
    
    print("Info")
    print(df.info())
    print("")
    
    print("Null check")
    print(df.isnull().sum())
    print("")
    
    print("Columns")
    print(columns)
    print("")
    
    print("Dataframe data types")
    print(df.dtypes)
    print("")
    
    print("Column types")
    print(col_types)
    print("")
    
    print("Categorical Columns")
    print(cat_cols)
    print("")
    
    print("Sample data")
    print(df.sample(5))
    print("")
    
    return df, col_types, cat_cols
# correlation plot

def corr_heatmap(df):
    # asusming you have a dataframe df
    correlations = df.corr()

    # Generate a mask for the upper triangle
    mask = np.zeros_like(correlations, dtype=np.bool)
    # array([[False, False],
    #       [False, False]])
    
    mask[np.triu_indices_from(mask)] = True
    # array([[ True,  True],
    #       [False,  True]])
    
    # Set up the matplotlib figure
    f, ax = plt.subplots(figsize=(11, 7))
    
    # Generate a custom diverging colormap
    cmap = sns.diverging_palette(220, 10, as_cmap=True)
    
    # Draw the heatmap with the mask and correct aspect ratio
    sns.heatmap(correlations, mask=mask, cmap=cmap, vmax=.3, center=0,
                square=True, linewidths=.5, cbar_kws={"shrink": .5})
    plt.show()
# helper to treat missing values

# Create table for missing data analysis
def draw_missing_data_table(df):
    total = df.isnull().sum().sort_values(ascending=False)
    percent = (df.isnull().sum()/df.isnull().count()).sort_values(ascending=False)
    missing_data = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])
    return missing_data
# Plot learning curve
def plot_learning_curve(estimator, title, X, y, ylim=None, cv=None,
                        n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):
    plt.figure()
    plt.title(title)
    if ylim is not None:
        plt.ylim(*ylim)
    plt.xlabel("Training examples")
    plt.ylabel("Score")
    train_sizes, train_scores, test_scores = learning_curve(
        estimator, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)
    train_scores_mean = np.mean(train_scores, axis=1)
    train_scores_std = np.std(train_scores, axis=1)
    test_scores_mean = np.mean(test_scores, axis=1)
    test_scores_std = np.std(test_scores, axis=1)
    plt.grid()

    plt.fill_between(train_sizes, train_scores_mean - train_scores_std,
                     train_scores_mean + train_scores_std, alpha=0.1,
                     color="r")
    plt.fill_between(train_sizes, test_scores_mean - test_scores_std,
                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
    plt.plot(train_sizes, train_scores_mean, 'o-', color="r",
             label="Training score")
    plt.plot(train_sizes, test_scores_mean, 'o-', color="g",
             label="Validation score")

    plt.legend(loc="best")
    return plt
# Plot validation curve
def plot_validation_curve(estimator, title, X, y, param_name, param_range, ylim=None, cv=None,
                        n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):
    train_scores, test_scores = validation_curve(estimator, X, y, param_name, param_range, cv)
    train_mean = np.mean(train_scores, axis=1)
    train_std = np.std(train_scores, axis=1)
    test_mean = np.mean(test_scores, axis=1)
    test_std = np.std(test_scores, axis=1)
    plt.plot(param_range, train_mean, color='r', marker='o', markersize=5, label='Training score')
    plt.fill_between(param_range, train_mean + train_std, train_mean - train_std, alpha=0.15, color='r')
    plt.plot(param_range, test_mean, color='g', linestyle='--', marker='s', markersize=5, label='Validation score')
    plt.fill_between(param_range, test_mean + test_std, test_mean - test_std, alpha=0.15, color='g')
    plt.grid() 
    plt.xscale('log')
    plt.legend(loc='best') 
    plt.xlabel('Parameter') 
    plt.ylabel('Score') 
    plt.ylim(ylim)<jupyter_output><empty_output><jupyter_text>---<jupyter_code># file details

f_directory = 'C:\\Users\\praveen\\Downloads\\'
f_name = 'Consumer_Complaints.csv'

# change directory
os.chdir(f_directory)

# check whether file is present in current directory
if not f_name in os.listdir():
    sys.exit()
# load dataframe

df, col_types, cat_cols = load_dataframe(f_name)
# missing values
draw_missing_data_table(df)
gc.collect()
# for this problem, we need only two columns
# Consumer complaint narrative and Product

req_col = ['Consumer complaint narrative', 'Product']

df = df[req_col]
# missing values
draw_missing_data_table(df)
df.sample(3)
# selecting only columns without missing values
df = df[pd.notnull(df['Consumer complaint narrative'])]
# missing values
draw_missing_data_table(df)
# encode values in Product column
df['Category_id'] = df['Product'].factorize()[0]

df.sample(4)
# category tf-idf
category_id_df = df[['Product', 'Category_id']].drop_duplicates().sort_values('Category_id')
category_id_df
# map product -> categories
category_to_id = dict(category_id_df.values)
category_to_id
id_to_category = dict(category_id_df[['Category_id', 'Product']].values)
id_to_category
df.head()
# full data
fig = plt.figure(figsize=(8,6))
df.groupby('Product')['Consumer complaint narrative'].count().plot.bar(ylim=0)
plt.show()
df.shape
# For learning purpose, we'll use a random sample of 1000 rows

df = df.sample(1000)
df.shape
fig = plt.figure(figsize=(8,6))
df.groupby('Product')['Consumer complaint narrative'].count().plot.bar(ylim=0)
plt.show()<jupyter_output><empty_output><jupyter_text>Multi-Class Classifier: Features and Design1. To train supervised classifiers, we first transformed the “Consumer complaint narrative” into a vector of numbers. We explored vector representations such as TF-IDF weighted vectors.2. After having this vector representations of the text we can train supervised classifiers to train unseen “Consumer complaint narrative” and predict the “product” on which they fall.<jupyter_code># tf-idf
tfidf = TfidfVectorizer(sublinear_tf=True, min_df=5, norm='l2', 
                        encoding='latin-1', ngram_range=(1, 2), stop_words='english')

tfidf
features = tfidf.fit_transform(df['Consumer complaint narrative']).toarray()
features.shape
labels = df['Category_id']
labels.shape<jupyter_output><empty_output><jupyter_text>Correlation between words in features<jupyter_code>N = 2
for Product, category_id in sorted(category_to_id.items()):
    features_chi2 = chi2(features, labels==category_id)
    indices = np.argsort(features_chi2[0])
    feature_names = np.array(tfidf.get_feature_names())[indices]
    unigrams = [v for v in feature_names if len(v.split(' ')) == 1]
    bigrams = [v for v in feature_names if len(v.split(' ')) == 2]
    print("# '{}':".format(Product))
    print("  . Most correlated unigrams:\n. {}".format('\n. '.join(unigrams[-N:])))
    print("  . Most correlated bigrams:\n. {}".format('\n. '.join(bigrams[-N:])))
gc.collect()<jupyter_output><empty_output><jupyter_text>Modeling<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(df['Consumer complaint narrative'], df['Product'], random_state = 0)
X_train.shape
count_vect = CountVectorizer()
count_vect
X_train_counts = count_vect.fit_transform(X_train)
X_train_counts
tfidf_transformer = TfidfTransformer()
tfidf_transformer
X_train_tfidf = tfidf_transformer.fit_transform(X_train_counts)
X_train_tfidf
clf = MultinomialNB().fit(X_train_tfidf, y_train)
clf
# sample prediction
print(clf.predict(count_vect.transform(["This company refuses to provide me verification and validation of debt per my right under the FDCPA. I do not believe this debt is mine."])))<jupyter_output>['Debt collection']
{'fraction_non_alphanumeric': 0.1221287456281312, 'fraction_numerical': 0.01087059268361849, 'mean_word_length': 3.927806241266884, 'pattern_counts': {'":': 1, '<': 20, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '431485', 'license': 'MIT', 'n_tokens_mistral': 3615, 'n_tokens_neox': 3468, 'n_words': 853, 'path': 'projects/classify/ml_multi_class_text_classification_1.ipynb', 'repo_name': 'futureseadev/hgwxx7', 'size': '10579'}