<jupyter_start><jupyter_text>Simple Approaches to Multi-Label ClassificationThis program is modified from https://github.com/nkartik94/Multi-Label-Text-Classificationon 2019/11/18 by <NAME>  1. EDA: Exploratory Data Analysis<jupyter_code>import os, sys, time
import csv
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from IPython.display import Markdown, display
def printmd(string):
    display(Markdown(string))
#printmd('**bold**')
data_path  = "../mlabel_corpora/JokeHumorLevel_OneHot.txt"
train_path = "../mlabel_corpora/JokeHumorLevel_train_OneHot_random.txt"
test_path  = "../mlabel_corpora/JokeHumorLevel_test_OneHot.txt"
# set global variables: df
df = pd.read_csv(data_path, delimiter="\t")
train_random = pd.read_csv(train_path, delimiter="\t")
test = pd.read_csv(test_path, delimiter="\t")
print(df.shape, train_random.shape, test.shape) # same as data_raw.shape in Jupyter<jupyter_output>(3365, 9) (1691, 9) (1674, 9)<jupyter_text>1.1. Checking for missing values<jupyter_code>missing_values_check = df.isnull().sum()
print(missing_values_check)<jupyter_output>ID            0
Title         0
Content       0
HumorLevel    0
1             0
2             0
3             0
4             0
5             0
dtype: int64<jupyter_text>1.2. Calculating number of jokes under each label<jupyter_code># Jokes with no label are considered to be clean jokes.
# Creating seperate column in dataframe to identify clean jokes.
# We use axis=1 to count row-wise and axis=0 to count column wise
def print_empty_label(df, s):
    rowSums = df.iloc[:,3:].sum(axis=1)
    #print(rowSums.shape)
    #print(rowSums.head())
    clean_comments_count = (rowSums==0).sum(axis=0)

    print(f"Total number of {s} jokes = ",len(df))
    print(f"Number of clean jokes in {s}= ",clean_comments_count)
    print(f"Number of {s} jokes with labels =",(len(df)-clean_comments_count))
    print()
print_empty_label(df, 'all')
print_empty_label(train_random, 'all')
print_empty_label(test, 'all')
# set global variables: categories
categories = list(df.columns.values)
print(categories)
categories = categories[4:]
print(categories)
# Calculating number of jokes in each category
def print_category_count(df, categories):
    counts = []
    for category in categories:
        counts.append((category, df[category].sum()))
    df_stats = pd.DataFrame(counts, columns=['category', 'number of jokes'])
    print(df_stats)
    print()
print_category_count(df, categories)
print_category_count(train_random, categories)
print_category_count(test, categories)
def plot_category_count(df, categories):
    sns.set(font_scale = 2)
    plt.figure(figsize=(15,8))

    ax= sns.barplot(categories, df.iloc[:,3:].sum().values)

    plt.title("Jokes in each category", fontsize=24)
    plt.ylabel('Number of jokes', fontsize=18)
    plt.xlabel('Joke Skill', fontsize=18)

    #adding the text labels
    rects = ax.patches
    #print(rects)
    labels = df.iloc[:,3:].sum().values
    #print(labels)
    for rect, label in zip(rects, labels):
        height = rect.get_height()
        ax.text(rect.get_x() + rect.get_width()/2, height + 5, label, ha='center', va='bottom', fontsize=18)

    plt.show()
#plot_category_count(df, categories)<jupyter_output><empty_output><jupyter_text>1.3. Calculating number of jokes having multiple labels<jupyter_code>def plot_multiple_label(mlc_labels, multiLabel_counts):
    sns.set(font_scale = 2)
    plt.figure(figsize=(15,8))

    ax = sns.barplot(mlc_labels, multiLabel_counts.values)

    plt.title("Jokes having multiple labels ")
    plt.ylabel('Number of jokes', fontsize=18)
    plt.xlabel('Number of labels', fontsize=18)

    #adding the text labels
    rects = ax.patches
    labels = multiLabel_counts.values
    for rect, label in zip(rects, labels):
        height = rect.get_height()
        ax.text(rect.get_x() + rect.get_width()/2, height + 5, label, ha='center', va='bottom')

    plt.show()
def print_multiple_label(df):
    rowSums = df.iloc[:,4:].sum(axis=1)
    multiLabel_counts = rowSums.value_counts()
    print(multiLabel_counts)
    multiLabel_counts = multiLabel_counts.iloc[:]
    #print(multiLabel_counts.index)
    mlc_labels = ['L'+str(i) for i in multiLabel_counts.index]
    print(mlc_labels)
    
    plot_multiple_label(mlc_labels, multiLabel_counts)
    ##return(mlc_labels, multiLabel_counts)
print_multiple_label(df)<jupyter_output>1    3365
dtype: int64
['L1']<jupyter_text>1.4. WordCloud representation of most used words in each category of jokes<jupyter_code>#!pip install wordcloud
from wordcloud import WordCloud, STOPWORDS
import jieba
import Stopwords
plt.figure(figsize=(40,25))

def MyWordCloud(plt, df, field, position):
    #subset = df[df.Pun==1]
    subset = df.loc[df[field] == 1] # https://stackoverflow.com/questions/17071871/how-to-select-rows-from-a-dataframe-based-on-column-values
    #print(subset.head()); exit
    text = str(subset.Content.values)
    words_list = jieba.lcut(Stopwords.clean_text(text))
    text = Stopwords.clean_words(words_list)
    cloud = WordCloud(
                          #stopwords=STOPWORDS,
                          stopwords=Stopwords.STOP_WORDS,
                          background_color='black',
                          font_path='SNsanafonGyou.ttf', # OSError: unknown file format
                          collocations=False,
                          width=2500,
                          height=1800
                         ).generate(" ".join(text))

    plt.subplot(3, 3, position)
    plt.axis('off')
    plt.title(field, fontsize=40)
    plt.imshow(cloud)
    return plt

'''
for i, c in enumerate(categories):
    plt = MyWordCloud(plt, df, c, i+1)

#plt.show()
'''<jupyter_output><empty_output><jupyter_text>2. Data Pre-Processing<jupyter_code># Compute statistics of the dataset: MaxLength, MinLength, AvgChars, AvgWords
Len = df.Content.map(len)
print(f'Number of characters in   all jokes: Max={max(Len)}, Min={min(Len)}, Avg={sum(Len)/len(Len)}')
Len = train_random.Content.map(len)
print(f'Number of characters in train jokes: Max={max(Len)}, Min={min(Len)}, Avg={sum(Len)/len(Len)}')
Len = test.Content.map(len)
print(f'Number of characters in  test jokes: Max={max(Len)}, Min={min(Len)}, Avg={sum(Len)/len(Len)}')
# set global variables: data
data = df
#data = df.loc[np.random.choice(df.index, size=3365)]
data.shape
import nltk
from nltk.corpus import stopwords
from nltk.stem.snowball import SnowballStemmer
import re

import sys
import warnings

if not sys.warnoptions:
    warnings.simplefilter("ignore")<jupyter_output><empty_output><jupyter_text>2.1. Cleaning Data<jupyter_code>def cleanHtml(sentence):
    cleanr = re.compile('<.*?>')
    cleantext = re.sub(cleanr, ' ', str(sentence))
    return cleantext

def cleanPunc(sentence): #function to clean the word of any punctuation or special characters
    cleaned = re.sub(r'[?|!|\'|"|#]',r'',sentence)
    cleaned = re.sub(r'[.|,|)|(|\|/]',r' ',cleaned)
    cleaned = cleaned.strip()
    cleaned = cleaned.replace("\n"," ")
    return cleaned

def keepAlpha(sentence):
    alpha_sent = ""
    for word in sentence.split():
        alpha_word = re.sub('[^a-z A-Z]+', ' ', word)
        alpha_sent += alpha_word
        alpha_sent += " "
    alpha_sent = alpha_sent.strip()
    return alpha_sent
import Stopwords # import my own module with STOP_WORDS
from nltk.stem import PorterStemmer, WordNetLemmatizer
ps = PorterStemmer()
wnl = WordNetLemmatizer()
def clean_text(text): 
    '''
    Given a raw text string, return a clean text string.
    Example: 
        input:  "Years  passed. 多少   年过 去 了 。  "
        output: "years passed.多少年过去了。"
    '''
    text = str(text)
    text = text.lower() # 'years  passed. 多少   年过 去 了 。'
    # Next line will remove redundant white space for jeiba to cut
    text = re.sub(r'\s+([^a-zA-Z0-9.])', r'\1', text) # years passed.多少年过去了。
# see: https://stackoverflow.com/questions/16720541/python-string-replace-regular-expression
    text = text.strip(' ')
    return text

def clean_words(text, RmvStopWord=True, RmvMark=True):
    words = jieba.lcut(text)
#    print("After jieba.lcut():", words)
#    WL = [ w 
    WL = [ ps.stem(w)
#    WL = [ wnl.lemmatize(w)
        for w in words 
          if (not re.match(r'\s', w)) # remove white spaces
            and (RmvMark==False or not re.match(r'\W', w)) # remove punctuations
#            and (RmvMark==False or not re.match('^[a-z_]$', w)) # remove punctuations
#            and (RmvMark==False or w not in PUNCTUATIONS)
            and (RmvStopWord==False or w not in Stopwords.STOP_WORDS)
            and (not re.match(r'^\d+$', w)) # remove digit
         ]
    WL = " ".join(WL)
    return WL
print(data.head())
data['Content'] = data['Content'].str.lower()
#data['Content'] = data['Content'].apply(cleanHtml)
#data['Content'] = data['Content'].apply(cleanPunc)
#data['Content'] = data['Content'].apply(keepAlpha)
data['Content'] = data['Content'].apply(clean_text)
data['Content'] = data['Content'].apply(clean_words)
#data.head()<jupyter_output>Building prefix dict from the default dictionary ...
Loading model from cache /var/folders/jh/lksld56x4_10wxpq687st0x80000gn/T/jieba.cache<jupyter_text>2.2. Removing Stop Words<jupyter_code>stop_words = set(stopwords.words('english'))
stop_words.update(['zero','one','two','three','four','five','six','seven','eight','nine','ten','may','also','across','among','beside','however','yet','within'])
re_stop_words = re.compile(r"\b(" + "|".join(stop_words) + ")\\W", re.I)
def removeStopWords(sentence):
    global re_stop_words
    return re_stop_words.sub(" ", sentence)

data['Content'] = data['Content'].apply(removeStopWords)
#data.head()<jupyter_output><empty_output><jupyter_text>2.3. Stemming<jupyter_code>stemmer = SnowballStemmer("english")
def stemming(sentence):
    stemSentence = ""
    for word in sentence.split():
        stem = stemmer.stem(word)
        stemSentence += stem
        stemSentence += " "
    stemSentence = stemSentence.strip()
    return stemSentence

data['Content'] = data['Content'].apply(stemming)
data.head()<jupyter_output><empty_output><jupyter_text>2.4. Train-Test Split<jupyter_code>from sklearn.model_selection import train_test_split

# set global variables: train, test
#train, test = train_test_split(data, random_state=42, test_size=0.30, shuffle=True)
train, test = train_test_split(data, random_state=42, train_size=1691, shuffle=False)

print(train.shape)
print(test.shape)
# set global variables: train_text, test_text
train_text = train['Content']
test_text = test['Content']
# Compute statistics of the dataset: MaxLength, MinLength, AvgChars, AvgWords
Len = data.Content.map(lambda x: len(x.split()))
print(f'Number of words in   all jokes: Max={max(Len)}, Min={min(Len)}, Avg={sum(Len)/len(Len)}')
Len = train.Content.map(lambda x: len(x.split()))
print(f'Number of words in train jokes: Max={max(Len)}, Min={min(Len)}, Avg={sum(Len)/len(Len)}')
Len = test.Content.map(lambda x: len(x.split()))
print(f'Number of words in  test jokes: Max={max(Len)}, Min={min(Len)}, Avg={sum(Len)/len(Len)}')<jupyter_output>Number of words in   all jokes: Max=491, Min=3, Avg=41.621991084695395
Number of words in train jokes: Max=491, Min=3, Avg=40.33234772324069
Number of words in  test jokes: Max=290, Min=4, Avg=42.924731182795696<jupyter_text>2.5. TF-IDF<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer
#vectorizer = TfidfVectorizer(strip_accents='unicode', analyzer='word', ngram_range=(1,3), norm='l2')
vectorizer = TfidfVectorizer(strip_accents='unicode', analyzer='word', 
                             ngram_range=(1,2), norm='l2')
vectorizer.fit(train_text)
vectorizer.fit(test_text)

x_train = vectorizer.transform(train_text)
y_train = train_random.drop(labels = ['ID', 'Title', 'Content', 'HumorLevel'], axis=1)

x_test = vectorizer.transform(test_text)
y_test = test.drop(labels = ['ID', 'Title', 'Content', 'HumorLevel'], axis=1)
time_TfidfVector = time.time()

def Create_TFxIDF(data_text, train_text, test_text):

# word level tf-idf
    #tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\w{1,}', max_features=10000)
    tfidf_vect = TfidfVectorizer(analyzer='word', token_pattern=r'\w{1,}', 
        stop_words=Stopwords.STOP_WORDS, max_df=0.95, min_df=2, max_features=10000)
    tfidf_vect.fit(data_text)
    xtrain_tfidf = tfidf_vect.transform(train_text)
    xtest_tfidf = tfidf_vect.transform(test_text)
    print(f"xtrain_tfidf.shape:{xtrain_tfidf.shape}, xtest_tfidf.shape: {xtest_tfidf.shape}")

# word level ngram tf-idf 
    tfidf_vect_ngram = TfidfVectorizer(analyzer='word', token_pattern=r'\w{1,}', 
                    stop_words=Stopwords.STOP_WORDS, max_df=0.95, min_df=2,
                    ngram_range=(2,3), max_features=10000)
    tfidf_vect_ngram.fit(data_text)
    xtrain_tfidf_ngram =  tfidf_vect_ngram.transform(train_text)
    xtest_tfidf_ngram =  tfidf_vect_ngram.transform(test_text)
    print(f"xtrain_tfidf_ngram.shape:{xtrain_tfidf.shape}, xtest_tfidf_ngram.shape: {xtest_tfidf.shape}")

# character level ngram tf-idf
    tfidf_vect_ngram_chars = TfidfVectorizer(analyzer='char', token_pattern=r'\w{1,}', 
                    stop_words=Stopwords.STOP_WORDS, max_df=0.95, min_df=2,
                    ngram_range=(2,3), max_features=10000)
    tfidf_vect_ngram_chars.fit(data_text)
    xtrain_tfidf_ngram_chars =  tfidf_vect_ngram_chars.transform(train_text) 
    xtest_tfidf_ngram_chars =  tfidf_vect_ngram_chars.transform(test_text) 
    print(f"xtrain_tfidf_ngram_chars.shape:{xtrain_tfidf.shape}, xtest_tfidf_ngram_chars.shape: {xtest_tfidf.shape}")

    print("It takes %4.2f seconds to convert 3 TFxIDF vectors."%(time.time()-time_TfidfVector))

    return (xtrain_tfidf, xtest_tfidf, 
             xtrain_tfidf_ngram, xtest_tfidf_ngram,
             xtrain_tfidf_ngram_chars, xtest_tfidf_ngram_chars,
            tfidf_vect, tfidf_vect_ngram, tfidf_vect_ngram_chars)

(xtrain_tfidf, xtest_tfidf, 
 xtrain_tfidf_ngram, xtest_tfidf_ngram,
 xtrain_tfidf_ngram_chars, xtest_tfidf_ngram_chars,
 tfidf_vect, tfidf_vect_ngram, tfidf_vect_ngram_chars) = Create_TFxIDF(data.Content, train_text, test_text)
# re-assign x_train and x_test to what we want
x_train, x_test, vectorizer = xtrain_tfidf, xtest_tfidf, tfidf_vect
#x_train, x_test, vectorizer = xtrain_tfidf_ngram, xtest_tfidf_ngram, tfidf_vect_ngram
#x_train, x_test, vectorizer = xtrain_tfidf_ngram_chars, xtest_tfidf_ngram_chars, tfidf_vect_ngram_chars
print(x_train.shape, x_test.shape)
#print(x_train)<jupyter_output>(1691, 9559) (1674, 9559)<jupyter_text>3. Multi-Label Classification  3.1. Multiple Binary Classifications - (One Vs Rest Classifier)<jupyter_code>from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
from sklearn.multiclass import OneVsRestClassifier

from sklearn.metrics import accuracy_score, roc_auc_score, roc_curve
from sklearn.metrics import precision_recall_fscore_support
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
# define the evaluation metrics
def tcfunc(x, n=4): # trancate a number to have n decimal digits
    d = '0' * n
    d = int('1' + d)
# https://stackoverflow.com/questions/4541155/check-if-a-number-is-int-or-float
    if isinstance(x, (int, float)): return int(x * d) / d
    return x

def print_cls_report(y_true, prediction):
    print('Test accuracy is %1.4f'%(accuracy_score(y_true, prediction)))

    print(classification_report(y_true, prediction))
    
    # http://scikit-learn.org/stable/modules/generated/sklearn.metrics.precision_recall_fscore_support.html
    print("\tPrecision\tRecall\tF1\tSupport")
    (Precision, Recall, F1, Support) = list(map(tcfunc, 
        precision_recall_fscore_support(y_true, prediction, average='micro')))
    print("Micro\t{}\t{}\t{}\t{}".format(Precision, Recall, F1, Support))

    (Precision, Recall, F1, Support) = list(map(tcfunc, 
        precision_recall_fscore_support(y_true, prediction, average='macro')))
    print("Macro\t{}\t{}\t{}\t{}".format(Precision, Recall, F1, Support))
    
#    if True:
    if False:
        print(confusion_matrix(y_true, prediction))
        try: 
            print(classification_report(y_true, prediction, digits=4))
        except ValueError:
            print('May be some category has no predicted samples')
        show_confusion_matrix(prediction)


    print(f'y_true.shape={y_true.shape}, prediction.shape={prediction.shape}')
    #print(y_true.head())
    #print(prediction[0:6])

    # https://stackoverflow.com/questions/152580/whats-the-canonical-way-to-check-for-type-in-python
    pred = prediction
    if not isinstance(pred, np.ndarray): pred = prediction.toarray()
    print(type(y_true), type(prediction), type(pred))
    try:
        # https://scikit-learn.org/stable/modules/generated/sklearn.metrics.roc_auc_score.html
        print('macro roc_auc_score is %1.4f'%(roc_auc_score(y_true, pred, average='macro'))) # default average=’macro’
        print('micro roc_auc_score is %1.4f'%(roc_auc_score(y_true, pred, average='micro')))
    except:
        print("roc_auc_score error!!!")
    try:
        fpr, tpr, thresholds = roc_curve(y_true, pred)
        print(f'fpr={fpr}\ntpr={tpr}\nthresholds={thresholds}')
    except:
        print('roc_curve error!!!')
%%time

# Using pipeline for applying logistic regression and one vs rest classifier
LogReg_pipeline = Pipeline([
                ('clf', OneVsRestClassifier(LogisticRegression(solver='sag'), n_jobs=-1)),
            ])

#for category in categories:
for category in categories:
    printmd('**Processing {} jokes...**'.format(category))
    
    # Training logistic regression model on train data
    LogReg_pipeline.fit(x_train, train_random[category])
    
    prediction = LogReg_pipeline.predict(x_test)
    print_cls_report(test[category], prediction)<jupyter_output><empty_output><jupyter_text>3.2. Multiple Binary Classifications - (Binary Relevance)<jupyter_code># To use binary relevance, run "pip install scikit-multilearn" in advance
from skmultilearn.problem_transform import BinaryRelevance
from sklearn.naive_bayes import GaussianNB
# Next line refers to: http://scikit.ml/tutorial.html
from sklearn.svm import SVC, LinearSVC
%%time

# initialize binary relevance multi-label classifier
#   with a gaussian naive bayes base classifier
classifier = BinaryRelevance(GaussianNB())

# train
classifier.fit(x_train, y_train)

# predict
predictions = classifier.predict(x_test)

printmd('**BinaryRelevance with GaussianNB()**')
print_cls_report(y_test, predictions)
%%time

# Next line refers to: http://scikit.ml/tutorial.html and 
#   http://scikit.ml/api/skmultilearn.problem_transform.br.html
#classifier = BinaryRelevance(classifier=SVC(), require_dense=[False, True]) # 0.5 very bad!
# https://scikit-learn.org/stable/modules/svm.html#unbalanced-problems
classifier = BinaryRelevance(classifier=LinearSVC(class_weight='balanced')) # 0.5314
#classifier = BinaryRelevance(classifier=LinearSVC()) # Test roc_auc_score is 0.5234

# train
classifier.fit(x_train, y_train)

# predict
predictions = classifier.predict(x_test)

printmd('**BinaryRelevance with LinearSVC()**')
print_cls_report(y_test, predictions)<jupyter_output><empty_output><jupyter_text>3.3. Classifier Chains<jupyter_code># using classifier chains
from skmultilearn.problem_transform import ClassifierChain
from sklearn.linear_model import LogisticRegression
%%time

# initialize classifier chains multi-label classifier
#classifier = ClassifierChain(LogisticRegression()) # Test roc_auc_score is 0.5159
classifier = ClassifierChain(LinearSVC(class_weight='balanced')) #  0.5327

# Training logistic regression model on train data
classifier.fit(x_train, y_train)

# predict
predictions = classifier.predict(x_test)

printmd('**Classifier Chains with LinearSVM()**')
print_cls_report(y_test, predictions)<jupyter_output><empty_output><jupyter_text>3.4. Label Powerset<jupyter_code># using Label Powerset
from skmultilearn.problem_transform import LabelPowerset
%%time

# initialize label powerset multi-label classifier
#classifier = LabelPowerset(LogisticRegression()) # Test roc_auc_score is 0.5059
classifier = LabelPowerset(LinearSVC(class_weight='balanced')) # 0.5541
# Test roc_auc_score is 0.5312 if xtrain_tfidf_ngram, xtest_tfidf_ngram are used.
# Test roc_auc_score is 0.5474 if xtrain_tfidf_ngram_chars, xtest_tfidf_ngram_chars are used

# train
classifier.fit(x_train, y_train)

# predict
predictions = classifier.predict(x_test)

printmd('**Label Powerset with LinearSVM()**')
print_cls_report(y_test, predictions)
# with open('out/HumorLevel_True.txt', 'w') as outF:
#    outF.write(y_test.to_csv(sep='\t', index=False))

# https://stackoverflow.com/questions/36967666/transform-scipy-sparse-csr-to-pandas
# with open('out/HumorLevel_Pred.txt', 'w') as outF:
#    outF.write(pd.DataFrame(predictions.toarray(), columns=list(y_test.columns)).to_csv(sep='\t', index=False))<jupyter_output><empty_output><jupyter_text>3.5. Adapted Algorithm<jupyter_code># http://scikit.ml/api/api/skmultilearn.adapt.html#skmultilearn.adapt.MLkNN
from skmultilearn.adapt import MLkNN
from scipy.sparse import csr_matrix, lil_matrix
%%time

classifier_new = MLkNN(k=10)

# Note that this classifier can throw up errors when handling sparse matrices.

x_train = lil_matrix(x_train).toarray()
y_train = lil_matrix(y_train).toarray()
x_test = lil_matrix(x_test).toarray()

# train
classifier_new.fit(x_train, y_train)

# predict
predictions_new = classifier_new.predict(x_test)

printmd('**MLkNN**')
print_cls_report(y_test, predictions_new)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13054902503111604, 'fraction_numerical': 0.01816254091181487, 'mean_word_length': 4.72552124571127, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '152932', 'license': 'MIT', 'n_tokens_mistral': 8015, 'n_tokens_neox': 7622, 'n_words': 1496, 'path': 'tools/JokeHumorLevel_OneHot_random.ipynb', 'repo_name': 'SamTseng/Chinese_Humor_MultiLabeled', 'size': '21703'}