<jupyter_start><jupyter_text>31009 - Final Project - Data Clean Ada, <NAME><jupyter_code>import numpy as np  
import pandas as pd 
import re   
import nltk  
from nltk.corpus import stopwords           
from nltk.stem.porter import PorterStemmer
from collections import Counter  
import seaborn as sns 
import matplotlib.pyplot as plt     
from IPython.core.display import display, HTML  
import string
from tensorflow.keras.preprocessing.text import Tokenizer  
from tensorflow.keras.preprocessing.sequence import pad_sequences   
import tensorflow as tf 
from tqdm import tqdm  
dat = pd.read_csv("train.csv")
dat.head()<jupyter_output><empty_output><jupyter_text>1. Data Processing:<jupyter_code>## Helper functions
def Clean_URL(text):
    ## remove URL hyper links
    url = re.compile(r'https?://\S+|www\.\S+')
    return url.sub(r'',text)

def Clean_html(text):
    ## remove html tags
    html=re.compile(r'<.*?>')
    return html.sub(r'',text)

def Clean_emoji(text):
    ## remove emojis
    pattern = re.compile("["
                          
                           u"\U0001F600-\U0001F64F"  
                 
                           u"\U0001F300-\U0001F5FF"  
                          
                           u"\U0001F680-\U0001F6FF"  
                          
                           u"\U0001F1E0-\U0001F1FF"  
                          
                           u"\U00002702-\U000027B0"
                          
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    return pattern.sub(r'', text)

## Remove punctuations in the text
def Clean_punct(text):    
    table = str.maketrans('','',string.punctuation)
    return text.translate(table)

from spellchecker import SpellChecker
spell = SpellChecker()

## Spelling Checking
def Correct_spellings(text):
    corrected = []
    misspelled = spell.unknown(text.split())
    for word in text.split():
        if word in misspelled:
            corrected.append(spell.correction(word))
        else:
            corrected.append(word)
    return " ".join(corrected)<jupyter_output><empty_output><jupyter_text>Data Clean:<jupyter_code>#Apply all helper functions
dat['text'] = dat['text'].apply(lambda x : Clean_URL(x))
dat['text'] = dat['text'].apply(lambda x : Clean_html(x))
dat['text'] = dat['text'].apply(lambda x : Clean_emoji(x))
dat['text'] = dat['text'].apply(lambda x : Clean_punct(x))
dat['text'] = dat['text'].apply(lambda x : Correct_spellings(x))
dat.to_csv(r'Cleaned_Train.csv', index = False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11844660194174757, 'fraction_numerical': 0.03223300970873787, 'mean_word_length': 2.402906208718626, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '390222', 'license': 'MIT', 'n_tokens_mistral': 884, 'n_tokens_neox': 821, 'n_words': 166, 'path': '1.1_Data_Clean.ipynb', 'repo_name': 'adajing0101/Short-Text-Classification', 'size': '2581'}