<jupyter_start><jupyter_text>Library<jupyter_code>import pandas as pd
import matplotlib
import re
import matplotlib.pyplot as plt
import seaborn as sns
import nltk
import enchant
import jellyfish
from pattern.en import suggest
from time import time
import multiprocessing

print(f'Pandas version : {pd.__version__}')
print(f'Matplotlib version : {matplotlib.__version__}')
print(f'Seaborn version : {sns.__version__}')
print(f'NLTK version : {nltk.__version__}')
print(f'Enchant version : {enchant.__version__}')<jupyter_output>Pandas version : 1.0.3
Matplotlib version : 3.2.1
Seaborn version : 0.10.1
NLTK version : 3.5
Enchant version : 2.0.0<jupyter_text>Manage dataset<jupyter_code>df_a = pd.read_csv('./dataset/1_ocr_text/train_ocr_1.csv', header=None)
df_b = pd.read_csv('./dataset/1_ocr_text/train_ocr_2.csv', header=None)
df_c = pd.read_csv('./dataset/1_ocr_text/train_ocr_3.csv', header=None)
df_d = pd.read_csv('./dataset/1_ocr_text/train_ocr_4.csv', header=None)
df_d

df = pd.concat([df_a, df_b, df_c, df_d], axis=0, ignore_index=True)
df.columns = ['words']

def to_list(words):
    if words == '[]':
        return []
    else:
        words = words.strip('"')
        words = words.strip('[')
        words = words.strip(']')
        words = words.split(',')
        words = [w.lstrip(' ') for w in words]
        words = [w.strip("'") for w in words]
        words = [w for w in words if w != '']

        return words

df['words'] = df['words'].apply(to_list)
df

df_train = pd.read_csv('./dataset/0_original_csv/train.csv')
df_train

df = pd.concat([df_train, df], axis=1, ignore_index=True)
df.columns = ['filename', 'category', 'words']
df<jupyter_output><empty_output><jupyter_text>Analyse (Before text processing)<jupyter_code>total_images_with_words = 0

def check_word_exist(words):
    global total_images_with_words
    if len(words) > 0:
        total_images_with_words += 1

df['words'].map(check_word_exist)

print(f'Total images : {len(df.index)}')
print(f'Total images with words : {total_images_with_words} ({total_images_with_words / len(df.index)})')

dict_word = {}

def count_word_occurance(words):
    global dict_word
    for w in words:
        if w not in dict_word:
            dict_word[w] = 1
        else:
            dict_word[w] += 1

df['words'].map(count_word_occurance)

words_freq = []
for i in [1, 2, 3, 4, 5, 10, 15, 20, 25, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]:
    common_words_occurance = {k: v for k, v in dict_word.items() if v >= i}
    words_freq.append((i, len(common_words_occurance)))

words_freq = pd.DataFrame(words_freq, columns=['Frequency', 'Total'], dtype='int32')

sns.catplot(x='Frequency', y='Total', kind='bar',
            data=words_freq, height=8, aspect=1)<jupyter_output><empty_output><jupyter_text>Process text > Dynamic programming is used to save time  1. remove words with occurance less than 5<jupyter_code>words_list = list(dict_word.keys())
for word in words_list:
    if dict_word[word] < 5:
        del dict_word[word]

print(f'Total words after process 1 : {len(dict_word.keys())}')<jupyter_output>Total words after process 1 : 19269<jupyter_text>2. filter words 1. Remove non char 2. Check word length is > 3 3. Check word in english dict & not in english stopwords 4. Check word in indonesian dict & not in indonesian stopwords 5. Check possible typos with words in english dict & not in english stopwords 6. Check possible typos with words in indonesia dict & not in indonesian stopwords<jupyter_code>eng_stopwords = nltk.corpus.stopwords.words('english')
ind_stopwords = nltk.corpus.stopwords.words('indonesian')

eng_dict = enchant.Dict('en')
ind_dict = enchant.Dict('id_ID')

# prevent weird bug calling suggest for first time
try:
    suggest('cocoa')
except:
    pass

def suggest_word(word, lang):
    if lang == 'eng':
        word_suggestion = suggest(word)
        if 0.6 < word_suggestion[0][1] < 1:
            return word_suggestion[0][0]
        return None
    else:
        word_suggestion = ind_dict.suggest(word)

        ws_index = -1
        ws_distance = 3 # means max allowed distance = 2
        for i in range(len(word_suggestion)):
            curr_distance = jellyfish.levenshtein_distance(word, word_suggestion[i])
            if curr_distance < ws_distance:
                ws_index = i
                ws_distance = curr_distance
        if ws_index != -1:
            return word_suggestion[ws_index]
        return None

def filter_word(word):
    # 1
    word = re.sub('[^a-z]', '', word)
    # 2
    if len(word) < 4:
        return None
    # 3
    if eng_dict.check(word):
        if word not in eng_stopwords:
            return (word, word)
        return None
    # 4
    if ind_dict.check(word):
        if word not in ind_stopwords:
            return (word, word)
        return None
    # 5
    suggested_word = suggest_word(word, lang='eng')
    if suggested_word is not None:
        if suggested_word not in eng_stopwords:
            return (word, suggested_word)
        return None
    # 6
    # suggested_word = suggest_word(word, lang='ind')
    # if suggested_word is not None:
    #     if suggested_word not in ind_stopwords:
    #         return (word, suggested_word)
    #     return None
    return None

words_list = list(dict_word.keys())
with multiprocessing.Pool(multiprocessing.cpu_count()) as pool:
    START_TIME = time()

    words_memoization_list = pool.map(filter_word, words_list)

    END_TIME = time()
    print(f'Process 2 took {END_TIME - START_TIME} seconds')

words_memoization = {}
for word in words_memoization_list:
    if word is not None:
        words_memoization[word[0]] = word[1]

print(f'Total words after process 2 : {len(words_memoization.keys())}')<jupyter_output>Process 2 took 60.34605002403259 seconds
Total words after process 2 : 8358<jupyter_text>3. apply changes to dataframe<jupyter_code>def apply_memoization(words):
    global words_memoization
    new_words = []
    for w in words:
        if w in words_memoization:
            new_words.append(words_memoization[w])
    return new_words

df['words'] = df['words'].apply(apply_memoization)
df

df.to_csv('./dataset/2_with_cleaned_text/train.min.csv', index=False)<jupyter_output><empty_output><jupyter_text>Analyse (After text processing)<jupyter_code>total_images_with_words = 0

df['words'].map(check_word_exist)

print(f'Total images : {len(df.index)}')
print(f'Total images with words : {total_images_with_words} ({total_images_with_words / len(df.index)})')

dict_word = {}

df['words'].map(count_word_occurance)

words_freq = []
for i in [5, 10, 15, 20, 25, 50, 75, 100, 200, 300, 400, 500, 1000, 2000]:
    common_words_occurance = {k: v for k, v in dict_word.items() if v >= i}
    words_freq.append((i, len(common_words_occurance)))

words_freq = pd.DataFrame(words_freq, columns=['Frequency', 'Total'], dtype='int32')

sns.catplot(x='Frequency', y='Total', kind='bar',
            data=words_freq, height=8, aspect=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1217527833285755, 'fraction_numerical': 0.02569226377390808, 'mean_word_length': 3.744075829383886, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '124606', 'license': 'Unlicense', 'n_tokens_mistral': 2594, 'n_tokens_neox': 2413, 'n_words': 596, 'path': '02a_clean.ipynb', 'repo_name': 'leofire19/scl-2020-product-detection', 'size': '7006'}