<jupyter_start><jupyter_text>Chat intents Initial Data Exploration<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import sklearn
import re
import nltk
from nltk.probability import FreqDist
from nltk.tokenize import word_tokenize
from nltk.tokenize import WhitespaceTokenizer
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from nltk.stem import WordNetLemmatizer

pd.set_option("display.max_rows", 500)
pd.set_option("display.max_columns", 500)
pd.set_option("max_colwidth", 400)
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('wordnet')<jupyter_output>[nltk_data] Downloading package punkt to /Users/dave/nltk_data...
[nltk_data]   Package punkt is already up-to-date!
[nltk_data] Downloading package stopwords to /Users/dave/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!
[nltk_data] Downloading package wordnet to /Users/dave/nltk_data...
[nltk_data]   Package wordnet is already up-to-date!<jupyter_text>Load data<jupyter_code>data_full = pd.read_csv('../data/raw/train.csv')
data_full.sample(10)
len(data_full)
(data_full.category.value_counts()
                   .to_frame()
                   .reset_index()
                   .rename(columns={'index': 'intent', 'category': 'count'}))
data_full[data_full['category']=='card_swallowed']<jupyter_output><empty_output><jupyter_text>Take a sample of the larger dataset to use for much of the analysis.<jupyter_code>data_sample = data_full.sample(n=1000, random_state=123)
data_sample.category.value_counts()<jupyter_output><empty_output><jupyter_text>Clean data<jupyter_code>stop_words = set(stopwords.words('english'))

print(stop_words)
def clean_text(text):
    """
    Summary: turns text input string into list of cleaned word tokens
    
    Arguments:
        text: str of text
    
    Returns:
        lemmatized_words: str, lemmatized words from original text after cleaning
    """
    
    # remove numbers
    clean_text = re.sub(r'[0-9]+', '', text)
    
    # remove punctuation
    clean_text = re.sub(r'[^\w\s]', '', clean_text)
    
    # convert everything to lowercase
    clean_text = clean_text.lower()
    
    # tokenize
    wt = WhitespaceTokenizer()
    words = wt.tokenize(clean_text)
    
    # remove stop words
    cleaned_words = []
    for w in words:
        if w not in stop_words:
            cleaned_words.append(w)
            
    # lemmatize words
    wnl = WordNetLemmatizer()
    wnl_lemmatized_tokens = []
    for token in cleaned_words:
        wnl_lemmatized_tokens.append(wnl.lemmatize(token))
    
    lemmatized_words = ' '.join(wnl_lemmatized_tokens)
    
    return lemmatized_words
data_sample['cleaned_text'] = data_sample['text'].apply(lambda x: clean_text(x))
data_sample.head()
sample_text = ' '.join(list(data_sample['text'].values))
print(sample_text[0:400])
clean_sample_text = ' '.join(list(data_sample['cleaned_text'].values))
print(clean_sample_text[0:400])<jupyter_output>im worried card might lost mail long usually take arrive got charged fee shouldnt cash charge making withdrawal took money account earlier charged issue account dont see cheque deposit made yesterday please assist way people send money receive money change address still received new card anywhere cant use card want multiple currency account possible wouldnt atm give cash business eu add people acc<jupyter_text>Word distributions across all messages in sample<jupyter_code># tokenize
wt = WhitespaceTokenizer()
clean_tokens = wt.tokenize(clean_sample_text)

freq_dist = FreqDist(clean_tokens)
print(freq_dist)
freq_dist.most_common(500)
freq_dist.get('account')
fig, ax = plt.subplots(figsize=(12,8))
freq_dist.plot(20, cumulative=False)
plt.show()<jupyter_output><empty_output><jupyter_text>Save cleaned data<jupyter_code>data_sample.to_csv('../data/processed/data_sample.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10164271047227925, 'fraction_numerical': 0.009240246406570842, 'mean_word_length': 4.764792899408284, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '369249', 'license': 'MIT', 'n_tokens_mistral': 1323, 'n_tokens_neox': 1285, 'n_words': 295, 'path': 'notebooks/01-explore_dataset.ipynb', 'repo_name': 'dborrelli/chat-intents', 'size': '3896'}