<jupyter_start><jupyter_text>Basic text pre-processingThe following basic preprocessing will be examined in this notebook. A similar example for this study can be found at [Analytics Vidhya's Website](https://www.analyticsvidhya.com/blog/2018/02/the-different-methods-deal-text-data-predictive-python/).- Lower casing- Punctuation removal- Stopwords removal- Frequent words removal- Rare words removal- Spelling correction- Tokenization- Stemming- Lemmatization<jupyter_code># imports
import nltk
import numpy as np
import pandas as pd

from textblob import TextBlob
from textblob import Word
from nltk.stem import PorterStemmer
try:
    from nltk.corpus import stopwords
except:
    nltk.download('stopwords')
dataset = pd.read_csv('../../data/processed/yp_leilanis-lahaina-2_rws.csv')
dataset.head()

reviews = dataset.iloc[:,1]
reviews.head()<jupyter_output><empty_output><jupyter_text>Text Preprocessing Methods  Lower casing<jupyter_code>def lower_line(line):
    line_arr = [x.lower() for x in line.split()]
    return(' '.join(line_arr))<jupyter_output><empty_output><jupyter_text>Punctuation Removal<jupyter_code>def punctuation_line(line):
    line = line.replace("[^a-zA-Z#]", " ")
    # line = line.replace('[^\w\s]','')
    return(line)<jupyter_output><empty_output><jupyter_text>Short Words Removal<jupyter_code>def shortwords_line(line):
    # line.apply(lambda x: ' '.join([w for w in x.split() if len(w)>2]))
    return (' '.join([w for w in line.split() if len(w)>2]))<jupyter_output><empty_output><jupyter_text>Stopwords Removal<jupyter_code>def stopwords_line(line):
    stopwords_list = set(stopwords.words('english'))
    line = ' '.join(x for x in line.split() if x not in stopwords_list)
    return(line)<jupyter_output><empty_output><jupyter_text>Frequent Words Removal<jupyter_code>freq = pd.Series(' '.join(lines).split()).value_counts()[:10]

freq = list(freq.index)

def freqwords_line(line):
    line = " ".join(x for x in line.split() if x not in freq)
    return(line)<jupyter_output><empty_output><jupyter_text>Rare Words Removal<jupyter_code>rare = pd.Series(' '.join(lines).split()).value_counts()[-10:]

rare = list(rare.index)

def rarewords_line(line):
    line = " ".join(x for x in line.split() if x not in rare)<jupyter_output><empty_output><jupyter_text>Spelling Correction<jupyter_code>def spellcheck_line(line):
        return(str(TextBlob(line).correct()))<jupyter_output><empty_output><jupyter_text>Tokenization<jupyter_code>def tokenize_line(line):
    return(" ".join(TextBlob(str(line)).words))<jupyter_output><empty_output><jupyter_text>Stemming<jupyter_code>st = PorterStemmer()
def stemming_line(line):
    line = " ".join([st.stem(word) for word in line.split()])
    return(line)<jupyter_output><empty_output><jupyter_text>Lemmatization<jupyter_code>def lemnatize_line(line):
    line = " ".join([Word(word).lemmatize() for word in line.split()])
    return(line)<jupyter_output><empty_output><jupyter_text>Implementation on sample dataset<jupyter_code>line = "It started out really good.  Crab cakes were ok, \
beet salad ok even though I had no idea what the cheese was \
and then after 45 minutes, cold entree.  But you know how it is. \
You wait so long and then it's cold like the runner took his 15 minute \
break and forgot about you but your starving so you eat it anyways. \
That's how it was for me."
# lower
line = lower_line(line)
# punctuation
line = punctuation_line(line)
# stopwords
line = stopwords_line(line)
# freq
line = freqwords_line(line)
# shortwords
line = shortwords_line(line)
# rare
# line = rarewords_line(line)
# spelling
# line = spelling_line(line)
# tokenize
line = tokenize_line(line)
# stemming
line = stemming_line(line)
# lemmatization
line = lemnatize_line(line)

line
# tokenize
lines_arr = []
for line in reviews:
    # lower
    line = lower_line(line)
    # punctuation
    line = punctuation_line(line)
    # stopwords
    line = stopwords_line(line)
    # freq
    line = freqwords_line(line)
    # shortwords
    line = shortwords_line(line)
    # rare
    line = rarewords_line(line)
    # spelling
    # line = spelling_line(line)
    # tokenize
    line = tokenize_line(line)
    # stemming
    line = stemming_line(line)
    # lemmatization
    line = lemnatize_line(line)
    
    lines_arr.append(line)
print(lines[2])
print('--------')
print(lines_arr[2])
dataset['reviews'] = lines_arr
dataset.head()
dataset.to_csv('../../data/processed/yp_leilanis-lahaina-2_rws_1.csv', index=False)
dataset_new = pd.read_csv('../../data/processed/yp_leilanis-lahaina-2_rws_1.csv')
dataset_new.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12940921878381303, 'fraction_numerical': 0.005193681021423934, 'mean_word_length': 5.179144385026738, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 51, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '25765', 'license': 'MIT', 'n_tokens_mistral': 1673, 'n_tokens_neox': 1624, 'n_words': 348, 'path': 'notebooks/data-cleaning/1.0-ms-datacleaning.ipynb', 'repo_name': 'metinsenturk/semantic-analysis', 'size': '4621'}