<jupyter_start><jupyter_text><jupyter_code># Cleaning - sample
import requests

r = requests.get("https://news.ycombinator.com")
print(r.text)
import re
pattern = re.compile(r'<.*?>')
print(pattern.sub('', r.text))
from bs4 import BeautifulSoup

# Remove HTML tags 
soup = BeautifulSoup(r.text, "html5lib")
print(soup.get_text())
# Find all articles
summaries = soup.find_all("tr", class_="athing")
summaries[0]
# Extract title
summaries[0].find("a", class_="storylink").get_text().strip()
# Find all articles extract titles
articles = []
summaries = soup.find_all("tr", class_="athing")
for summary in summaries:
    title = summary.find("a", class_="storylink").get_text().strip()
    articles.append((title))

print(len(articles), "Article summaries found. Sample:")
print(articles[0])
# Normalization
# Case normalization
text = "The first time you see The Second Renaissance it may look boring. Look at it at least twice and definitely watch part 2. It will change your view of the matrix. Are the human people the ones who started the war ? Is AI a bad thing ?"
print(text)

# convert lower case
text = text.lower()
print(text)

# Punctuation removal
import re
text = re.sub(r"[^a-zA-Z0-9]"," ", text)
print(text)

# Tokenization
words = text.split()
print(words)<jupyter_output>The first time you see The Second Renaissance it may look boring. Look at it at least twice and definitely watch part 2. It will change your view of the matrix. Are the human people the ones who started the war ? Is AI a bad thing ?
the first time you see the second renaissance it may look boring. look at it at least twice and definitely watch part 2. it will change your view of the matrix. are the human people the ones who started the war ? is ai a bad thing ?
the first time you see the second renaissance it may look boring  look at it at least twice and definitely watch part 2  it will change your view of the matrix  are the human people the ones who started the war   is ai a bad thing  
['the', 'first', 'time', 'you', 'see', 'the', 'second', 'renaissance', 'it', 'may', 'look', 'boring', 'look', 'at', 'it', 'at', 'least', 'twice', 'and', 'definitely', 'watch', 'part', '2', 'it', 'will', 'change', 'your', 'view', 'of', 'the', 'matrix', 'are', 'the', 'human', 'people', 'the', 'ones', '[...]<jupyter_text>NLTK- Natural Language Toolkit<jupyter_code>import os
import re
import nltk
nltk.data.path.append(os.path.join(os.getcwd(), "nltk_data")) 
# Another sample text
text = "Dr. Smith graduated from the University of Washington. He later started an analytics firm called Lux, which catered to enterprise customers."
print(text)

# NLTK -tokenization
from nltk.tokenize import word_tokenize
# nltk.download('punkt')
words = word_tokenize(text)
print('>>>WORDS-TOKEN<<<',words)

# Split text into sentense
from nltk.tokenize import sent_tokenize
sentences = sent_tokenize(text)
print('>>>SENTENCES-TOKEN<<<',sentences)

# list of stop words
from nltk.corpus import stopwords
# nltk.download('stopwords')
print('>>>STOP-WORDS<<<', stopwords.words("english"))
# 
# Reset text
text = "The first time you see The Second Renaissance it may look boring. Look at it at least twice and definitely watch part 2. It will change your view of the matrix. Are the human people the ones who started the war ? Is AI a bad thing ?"
# Normalize it
text = re.sub(r"[^a-zA-Z0-9]"," ", text.lower())

# tokenize it
words = text.split()
print('>>>TOKENIZE<<< ',words)

# Remove stop words
words = [w for w in words if w not in stopwords.words("english")]
print('>>>STOP-WORDS-REMOVED<<< ',words)
# Sentence parsing
import nltk
my_grammer = nltk.CFG.fromstring("""
S -> NP VP
PP -> P NP
NP -> Det N | Det N PP | 'I'
VP -> V NP | VP PP
Det -> 'an' | 'my'
N -> 'elephant' | 'pajamas'
V -> 'shot'
P -> 'in'
""")

parser = nltk.ChartParser(my_grammer)

# Parse sentence 
sentence = word_tokenize("I shot an elephant in my pajamas")
for tree in parser.parse(sentence):
  print(tree)
# Stemming & Lemmatization
# Stemming
from nltk.stem.porter import PorterStemmer
# Reduce words to their stem
stemmed = [PorterStemmer().stem(w) for w in words]
print('>>>STEMMING<<< ', stemmed)
# Lemmatization
from nltk.stem.wordnet import WordNetLemmatizer
# nltk.download('wordnet')

# Reduce to their root form
lemmed = [WordNetLemmatizer().lemmatize(w) for w in words]
print('>>>LEMMATEING<<< ', stemmed)

# Lemmatize verbs by specifying pos
lemmed = [WordNetLemmatizer().lemmatize(w, pos='v') for w in lemmed]
print('>>>LEMMATING_POS<<< ', lemmed)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13646384479717813, 'fraction_numerical': 0.0033068783068783067, 'mean_word_length': 4.946264744429882, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 41, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '520745', 'license': 'MIT', 'n_tokens_mistral': 1526, 'n_tokens_neox': 1474, 'n_words': 551, 'path': 'Text_processing.ipynb', 'repo_name': 'iamnikhilpal/deeplearning-practice', 'size': '4536'}