<jupyter_start><jupyter_text>Text PreprocessingIn this notebook, you will practice how to preprocess text data before analyzing it. It's an inevitable process for every NLP project.Reference:    * https://github.com/gilbutITbook/080289/blob/main/chap09/colab_9%EC%9E%A5.ipynb<jupyter_code>!pip install konlpy
import numpy as np
import pandas as pd
import nltk
nltk.download("popular")
text=nltk.word_tokenize("Is it possible distinguishing cats and dogs")
text
nltk.download('averaged_perceptron_tagger')
nltk.pos_tag(text)
nltk.download('punkt')
string1="my favorite subject is math"
string2="my favorite subject is math, english, economic and computer science"
nltk.word_tokenize(string1)
nltk.word_tokenize(string2)
# KoNLPy
from konlpy.tag import Komoran
komoran = Komoran()
print(komoran.morphs('딥러닝이 쉽나요? 어렵나요?'))
print(komoran.pos('소파 위에 있는 것이 고양이인가요? 강아지인가요?'))<jupyter_output>[('소파', 'NNP'), ('위', 'NNG'), ('에', 'JKB'), ('있', 'VV'), ('는', 'ETM'), ('것', 'NNB'), ('이', 'JKS'), ('고양이', 'NNG'), ('이', 'VCP'), ('ㄴ가요', 'EF'), ('?', 'SF'), ('강아지', 'NNG'), ('이', 'VCP'), ('ㄴ가요', 'EF'), ('?', 'SF')]<jupyter_text>Tokenizing<jupyter_code># Sentence Tokenizing
from nltk import sent_tokenize
text_sample = 'Natural Language Processing, or NLP, is the process of extracting the meaning, or intent, behind human language. In the field of Conversational artificial intelligence (AI), NLP allows machines and applications to understand the intent of human language inputs, and then generate appropriate responses, resulting in a natural conversation flow.'
tokenized_sentences = sent_tokenize(text_sample)
print(tokenized_sentences)
# Word Tokenizing
from nltk import word_tokenize
sentence = " This book is for deep learning learners"
words = word_tokenize(sentence)
print(words)
# Tokenizing w.r.t. apostrophe
from nltk.tokenize import WordPunctTokenizer  
sentence = "it’s nothing that you don’t already know except most people aren’t aware of how their inner world works."
words = WordPunctTokenizer().tokenize(sentence)
print(words)
data_url = "https://raw.githubusercontent.com/gilbutITbook/080289/main/chap09/data/ratings_train.txt"
rating_data = pd.read_csv(data_url, sep='\t')
rating_data.head()
len(rating_data)
# 형태소 분석
from konlpy.tag import Okt
from tqdm import tqdm
# Twitter(Okt) - 오픈 소스 한글 형태소 분석기
okt = Okt()

result = []
for comment in tqdm(rating_data.document.values[:1000], desc='형태소 분석 중...'):
  malist = okt.pos(comment, norm=True, stem=True)
  r = []
  for word in malist:
    if not word[1] in 'Josa, Eomi, Punctuation'.split(', '):
      r.append(word[0])
  rl = (' '.join(r)).strip()
  result.append(rl)

result[:10]<jupyter_output><empty_output><jupyter_text>Removing Stopwords<jupyter_code>from nltk.corpus import stopwords
nltk.download('stopwords')
nltk.download('punkt')
from nltk.tokenize import word_tokenize

sample_text = "One of the first things that we ask ourselves is what are the pros and cons of any task we perform."
text_tokens = word_tokenize(sample_text)

tokens_without_sw = [word for word in text_tokens if not word in stopwords.words('english')]
print("불용어 제거 미적용:", text_tokens, '\n')
print("불용어 제거 적용:",tokens_without_sw)<jupyter_output><empty_output><jupyter_text>Stemming and Lemmatization ex) writing, writes, wrote, written -> write<jupyter_code># Porter algorithm
from nltk.stem import PorterStemmer
stemmer = PorterStemmer()

print(stemmer.stem('obesses'),stemmer.stem('obssesed'))
print(stemmer.stem('standardizes'),stemmer.stem('standardization'))
print(stemmer.stem('national'), stemmer.stem('nation'))
print(stemmer.stem('absentness'), stemmer.stem('absently'))
print(stemmer.stem('tribalical'), stemmer.stem('tribalicalized'))
from nltk.stem import LancasterStemmer
stemmer = LancasterStemmer()

print(stemmer.stem('obsesses'),stemmer.stem('obsessed'))
print(stemmer.stem('standardizes'),stemmer.stem('standardization'))
print(stemmer.stem('national'), stemmer.stem('nation'))
print(stemmer.stem('absentness'), stemmer.stem('absently'))
print(stemmer.stem('tribalical'), stemmer.stem('tribalicalized'))
# Lemmatization

nltk.download('wordnet')

from nltk.stem import WordNetLemmatizer
lemma = WordNetLemmatizer()

print(stemmer.stem('obsesses'),stemmer.stem('obsessed'))
print(lemma.lemmatize('standardizes'),lemma.lemmatize('standardization'))
print(lemma.lemmatize('national'), lemma.lemmatize('nation'))
print(lemma.lemmatize('absentness'), lemma.lemmatize('absently'))
print(lemma.lemmatize('tribalical'), lemma.lemmatize('tribalicalized'))
print(lemma.lemmatize('obsesses', 'v'),lemma.lemmatize('obsessed','a'))
print(lemma.lemmatize('standardizes','v'),lemma.lemmatize('standardization','n'))
print(lemma.lemmatize('national','a'), lemma.lemmatize('nation','n'))
print(lemma.lemmatize('absentness','n'), lemma.lemmatize('absently','r'))
print(lemma.lemmatize('tribalical','a'), lemma.lemmatize('tribalicalized','v'))<jupyter_output>obsess obsessed
standardize standardization
national nation
absentness absently
tribalical tribalicalized<jupyter_text>To-do Using requests and bs4 library, get today's lunch and dinner menu of '두레미담' then pre-process them.<jupyter_code>import requests
from bs4 import BeautifulSoup
url = "https://snuco.snu.ac.kr/foodmenu"
html = requests.get(url)
soup = BeautifulSoup(html.text, 'html.parser')
lunch = soup.find_all('td', {'class': 'views-field views-field-field-lunch'})
# 두레미담: 5th restaurant
lunch[4]
p_tag_list = lunch[4].find_all('p')

menus = []
for idx, item in enumerate(p_tag_list[1:]):
  menu = item.get_text()
  if menu != '\xa0':
    menus.append(menu)
  if menu == '오늘의차/그린샐러드':
    break
menus<jupyter_output><empty_output><jupyter_text>Now complete rest of the task<jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14840062926061878, 'fraction_numerical': 0.006642195420381052, 'mean_word_length': 7.4147058823529415, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 24, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '548378', 'license': 'MIT', 'n_tokens_mistral': 2155, 'n_tokens_neox': 2122, 'n_words': 453, 'path': 'Day 13/13_1_text_preprocessing.ipynb', 'repo_name': 'SLCFLAB/Data-Science-Python', 'size': '5721'}