<jupyter_start><jupyter_text>Configuration
____________________<jupyter_code># set main directories
BASE_DIR = '/content/gdrive/MyDrive/AML_project/project/'
UTILS_DIR = BASE_DIR + 'utils'
DATA_DIR = BASE_DIR + 'data/'
TOKENIZER_DIR = BASE_DIR + 'tokenizers/'
# mount drive
from google.colab import drive
drive.mount('/content/gdrive', force_remount=True)
# set utils directory
import sys
sys.path.append(UTILS_DIR)<jupyter_output>Mounted at /content/gdrive<jupyter_text>Imports
____________________<jupyter_code>import io
import numpy as np
import pandas as pd
import nltk
import itertools
import string
import re
import sys
import json
import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure
from collections import Counter
from nltk.corpus import stopwords
from wordcloud import WordCloud
from google.colab import drive
from itertools import cycle
from collections import OrderedDict
# custom imports
from preprocessing import Preprocessor
from utils import getComments<jupyter_output>[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.
[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Unzipping tokenizers/punkt.zip.<jupyter_text>Load data
____________________<jupyter_code># load original train
train = pd.read_csv(DATA_DIR + 'train.csv')<jupyter_output><empty_output><jupyter_text>Utils
____________________<jupyter_code># change width of bars
def change_width(ax, new_value) :
  for patch in ax.patches :
    current_width = patch.get_width()
    diff = current_width - new_value

    # we change the bar width
    patch.set_width(new_value)

    # we recenter the bar
    patch.set_x(patch.get_x() + diff * .5)

# plot a hist
def plotHist(x, y, title, orient='v'):
  sns.set_style("white")
  figure(figsize=(10,6))
  ax = sns.barplot(x=x, y=y, orient=orient)
  plt.xlabel('Labels', fontsize=18)
  plt.ylabel('Number of instances', fontsize=18)
  plt.title(title, fontsize=16)
  change_width(ax, 0.3)
  plt.show(ax)

# get wordcloud for some most words
def getWordCloud(most_common_words):
  wc = WordCloud()
  wc.generate_from_frequencies(frequencies=dict(most_common_words))
  sns.reset_orig()
  figure(figsize=(10,10))
  plt.axis("off")
  plt.imshow(wc)

# plot top n words
def plotMostCommonWords(most_common_words, n_words, title):
  val, cnt = (zip(*most_common_words))
  val, cnt = list(val[0:n_words]), list(cnt[0:n_words])
  sns.set_style("white")
  figure(figsize=(20,8))
  plt.bar(np.arange(n_words), cnt, label=True)
  plt.xticks(np.arange(len(cnt)), val)
  plt.xlabel('Top words', fontsize=18)
  plt.ylabel('Word frequency', fontsize=18)
  plt.title(title, fontsize=18)
  plt.show()

labels = ['toxic', 'severe_toxic',	'obscene',	'threat',	'insult',	'identity_hate']<jupyter_output><empty_output><jupyter_text>Data exploration
____________________  Let's see some of the data it contains<jupyter_code>train.head(10)<jupyter_output><empty_output><jupyter_text>Let's see some of the content of those comments<jupyter_code>p = Preprocessor(replace_emojis=False)

# preprocess text to remove irrelevant words
preprocessed_train = train.copy()
preprocessed_train['comment_text'] = preprocessed_train['comment_text'].map(p.preprocessDoc)

# filter toxic comments
toxic_comments = getComments(preprocessed_train, clean=False)['comment_text'].values


# filter non-toxic comments
non_toxic_comments = getComments(preprocessed_train, clean=True)['comment_text'].values

# tokenize documents
toxic_comments_tokens = [p.tokenize(doc) for doc in toxic_comments]
non_toxic_comments_tokens = [p.tokenize(doc) for doc in non_toxic_comments]

# flatten lists to compute most common words
toxic_comments_tokens = list(itertools.chain.from_iterable(toxic_comments_tokens))
non_toxic_comments_tokens = list(itertools.chain.from_iterable(non_toxic_comments_tokens))<jupyter_output><empty_output><jupyter_text>Let's visualize wordclouds for non toxic comments<jupyter_code>c = Counter(non_toxic_comments_tokens)
most_common_non_toxic = c.most_common()
getWordCloud(most_common_non_toxic)<jupyter_output><empty_output><jupyter_text>Let's also plot the most common words for this category<jupyter_code>plotMostCommonWords(most_common_non_toxic, 20, 'Most common non-toxic words')<jupyter_output><empty_output><jupyter_text>Let's visualize wordclouds for toxic comments<jupyter_code>c = Counter(toxic_comments_tokens)
most_common_toxic = c.most_common()
getWordCloud(most_common_toxic)<jupyter_output><empty_output><jupyter_text>Let's also plot the most common words for this category<jupyter_code>plotMostCommonWords(most_common_toxic, 20, 'Most common toxic words')<jupyter_output><empty_output><jupyter_text>Let's see the ratio between toxic and non toxic comments<jupyter_code>print('Number of non toxic instances: ', len(non_toxic_comments))
print('Number of toxic instances: ', len(toxic_comments))
plotHist(y=[len(non_toxic_comments), len(toxic_comments)], x=['Non toxic', 'Toxic'], title='Ratio Toxic / Non-toxic')<jupyter_output><empty_output><jupyter_text>As we can see the dataset is really unbalanced towards the non toxic examples. In fact toxic comments are only about `10%` of the training data.  Let's see the distribution of toxic labels<jupyter_code># sum the columns for each label
sum_labels = train[labels].sum(axis=0)
plotHist(y=sum_labels, x=sum_labels.index, title='Labels distribution')<jupyter_output><empty_output><jupyter_text>As we can see the labels distribution is also really unbalanced. Let's also see how many labels are indipendent from one another.<jupyter_code># filter instances for each category
# toxic
toxic = train[(train['toxic'] == 1) & (train['severe_toxic'] == 0) & (train['obscene'] == 0) & (train['threat'] == 0) & (train['insult'] == 0) & (train['identity_hate'] == 0)]
# severe toxic
severe_toxic = train[(train['toxic'] == 0) & (train['severe_toxic'] == 1) & (train['obscene'] == 0) & (train['threat'] == 0) & (train['insult'] == 0) & (train['identity_hate'] == 0)]
# obscene
obscene = train[(train['toxic'] == 0) & (train['severe_toxic'] == 0) & (train['obscene'] == 1) & (train['threat'] == 0) & (train['insult'] == 0) & (train['identity_hate'] == 0)]
# threat
threat = train[(train['toxic'] == 0) & (train['severe_toxic'] == 0) & (train['obscene'] == 0) & (train['threat'] == 1) & (train['insult'] == 0) & (train['identity_hate'] == 0)]
# insult
insult = train[(train['toxic'] == 0) & (train['severe_toxic'] == 0) & (train['obscene'] == 0) & (train['threat'] == 0) & (train['insult'] == 1) & (train['identity_hate'] == 0)]
# identity_hate
identity_hate = train[(train['toxic'] == 0) & (train['severe_toxic'] == 0) & (train['obscene'] == 0) & (train['threat'] == 0) & (train['insult'] == 0) & (train['identity_hate'] == 1)]
print('Toxic:', len(toxic))
print('Severe Toxic:', len(severe_toxic))
print('Obscene:', len(obscene))
print('Threat:', len(threat))
print('Insult:', len(insult))
print('Identity Hate:', len(identity_hate))<jupyter_output>Toxic: 5666
Severe Toxic: 0
Obscene: 317
Threat: 22
Insult: 301
Identity Hate: 54
{'fraction_non_alphanumeric': 0.13439667128987517, 'fraction_numerical': 0.012066574202496532, 'mean_word_length': 6.350662589194699, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '544872', 'license': 'MIT', 'n_tokens_mistral': 2643, 'n_tokens_neox': 2469, 'n_words': 554, 'path': 'data_exploration.ipynb', 'repo_name': 'christianbernasconi96/ToxicCommentChallenge', 'size': '7210'}