<jupyter_start><jupyter_text>Notebook for experimenting with text processing<jupyter_code># necessary to add root to the path to access core package
import sys
sys.path.append(sys.path[0] + '/..')
# print(sys.path)
import os
print(os.getcwd())
from typing import List, Dict

from IPython.display import Image

%load_ext autoreload
%autoreload 2
# Here add packages you develop
SAMPLE_TEXT = """
<p style='clear:both;'>Loris gypsum dolour sits is meat, dictum divides sentient per no sole. Requiem effendi and vies, sequel rill animal ea has, ad prod option integer ea. Use it mollies nostrum, per en appropriate negligent. Ala emus no duo, obi verger debus an. Ea elite ague disco bequeath eons. Gracie nominal set id xiv.</p>

<p style='clear:both;'>Pro id bonbon accustoms. Stet probates in duo. Set ponder um expedients cu, veil ex quid dictum momentum. Gracie nominal set id xiv. Eel cues linguist efficient ea, veil sale disciple at. Es drams vituperate it, amateur lucid lids ex mesh.</p>

<p style='clear:both;'>Brusque croquet pro ea. Deter ornate.</p>
"""<jupyter_output><empty_output><jupyter_text>Text processing using Regex and BeuatifulSoup* [BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/bs4/doc/)<jupyter_code>import re
from bs4 import BeautifulSoup
from sklearn.feature_extraction.text import CountVectorizer

REPLACE_NO_SPACE = re.compile("[.;:!\'?,\"()\[\]]")
REPLACE_WITH_SPACE = re.compile("(<br\s*/><br\s*/>)|(\-)|(\/)")

def clear_text(text):
    soup = BeautifulSoup(text)
    text = soup.get_text()
    text = text.replace('\n', ' ')
    text = REPLACE_NO_SPACE.sub("", text.lower())
    text = REPLACE_WITH_SPACE.sub(" ", text)
    return text

def split_text(text):
    return text.split()

print(clear_text(SAMPLE_TEXT))
# split_text(clear_text(SAMPLE_TEXT))<jupyter_output>loris gypsum dolour sits is meat dictum divides sentient per no sole requiem effendi and vies sequel rill animal ea has ad prod option integer ea use it mollies nostrum per en appropriate negligent ala emus no duo obi verger debus an ea elite ague disco bequeath eons gracie nominal set id xiv pro id bonbon accustoms stet probates in duo set ponder um expedients cu veil ex quid dictum momentum gracie nominal set id xiv eel cues linguist efficient ea veil sale disciple at es drams vituperate it amateur lucid lids ex mesh brusque croquet pro ea deter ornate<jupyter_text>Text processing using NLTK<jupyter_code>import nltk
# nltk.download() # Uncomment only if you do not the data already downlaoded
# Check nltk data by uncommenting following lines
from nltk.corpus import brown
brown.words()
from nltk.stem import PorterStemmer, WordNetLemmatizer
from nltk.corpus import wordnet
from nltk.stem import PorterStemmer, WordNetLemmatizer
from nltk.corpus import wordnet, stopwords

lemmatizer = WordNetLemmatizer()
stemmer = PorterStemmer()
stop_words = set(stopwords.words("english"))

def process_text(text):
    soup = BeautifulSoup(text)
    text = soup.get_text()
    words = nltk.word_tokenize(text)
    words = [lemmatizer.lemmatize(word, wordnet.VERB) for word in words]
    words = [word for word in words if not word in stop_words]
    return words

print(process_text(clear_text(SAMPLE_TEXT)))<jupyter_output>['loris', 'gypsum', 'dolour', 'sit', 'meat', 'dictum', 'divide', 'sentient', 'per', 'sole', 'requiem', 'effendi', 'vie', 'sequel', 'rill', 'animal', 'ea', 'ad', 'prod', 'option', 'integer', 'ea', 'use', 'mollies', 'nostrum', 'per', 'en', 'appropriate', 'negligent', 'ala', 'emus', 'duo', 'obi', 'verger', 'debus', 'ea', 'elite', 'ague', 'disco', 'bequeath', 'eons', 'gracie', 'nominal', 'set', 'id', 'xiv', 'pro', 'id', 'bonbon', 'accustom', 'stet', 'probate', 'duo', 'set', 'ponder', 'um', 'expedients', 'cu', 'veil', 'ex', 'quid', 'dictum', 'momentum', 'gracie', 'nominal', 'set', 'id', 'xiv', 'eel', 'cue', 'linguist', 'efficient', 'ea', 'veil', 'sale', 'disciple', 'es', 'drams', 'vituperate', 'amateur', 'lucid', 'lids', 'ex', 'mesh', 'brusque', 'croquet', 'pro', 'ea', 'deter', 'ornate']<jupyter_text>[Sklearn text feature extraction](https://scikit-learn.org/stable/modules/classes.htmlmodule-sklearn.feature_extraction.text)* [CountVectorizer](https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.htmlsklearn.feature_extraction.text.CountVectorizer)  Convert a collection of text documents to a matrix of token counts```This implementation produces a sparse representation of the counts using scipy.sparse.csr_matrix.```<jupyter_code>vectorizer = CountVectorizer()
X = vectorizer.fit_transform([clear_text(SAMPLE_TEXT)])
print(vectorizer.get_feature_names())
vectorizer = CountVectorizer()
X = vectorizer.fit_transform([' '.join(process_text(clear_text(SAMPLE_TEXT)))])
print(vectorizer.get_feature_names())<jupyter_output>['accustom', 'ad', 'ague', 'ala', 'amateur', 'animal', 'appropriate', 'bequeath', 'bonbon', 'brusque', 'croquet', 'cu', 'cue', 'debus', 'deter', 'dictum', 'disciple', 'disco', 'divide', 'dolour', 'drams', 'duo', 'ea', 'eel', 'effendi', 'efficient', 'elite', 'emus', 'en', 'eons', 'es', 'ex', 'expedients', 'gracie', 'gypsum', 'id', 'integer', 'lids', 'linguist', 'loris', 'lucid', 'meat', 'mesh', 'mollies', 'momentum', 'negligent', 'nominal', 'nostrum', 'obi', 'option', 'ornate', 'per', 'ponder', 'pro', 'probate', 'prod', 'quid', 'requiem', 'rill', 'sale', 'sentient', 'sequel', 'set', 'sit', 'sole', 'stet', 'um', 'use', 'veil', 'verger', 'vie', 'vituperate', 'xiv']<jupyter_text>Conclusion Following text clearning function was moved to lib folder:<jupyter_code>import re
from bs4 import BeautifulSoup
from nltk.stem import PorterStemmer, WordNetLemmatizer
from nltk.corpus import wordnet, stopwords

REPLACE_NO_SPACE = re.compile("[.;:!\'?,\"()\[\]]")
REPLACE_WITH_SPACE = re.compile("(<br\s*/><br\s*/>)|(\-)|(\/)")
LEMMATIZER = WordNetLemmatizer()
STEMMER = PorterStemmer()
ENGLISH_STOP_WORDS = set(stopwords.words("english"))

def clean_text_en(text):
    soup = BeautifulSoup(text)
    text = soup.get_text()
    text = text.replace('\n', ' ')
    text = REPLACE_NO_SPACE.sub("", text.lower())
    text = REPLACE_WITH_SPACE.sub(" ", text)
    words = nltk.word_tokenize(text)
    words = [lemmatizer.lemmatize(word, wordnet.VERB) for word in words]
    words = [word for word in words if not word in stop_words]
    return ' '.join(words)
clean_text_en(SAMPLE_TEXT)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15770609318996415, 'fraction_numerical': 0.0007791803023219573, 'mean_word_length': 5.976086956521739, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '565778', 'license': 'MIT', 'n_tokens_mistral': 2332, 'n_tokens_neox': 2256, 'n_words': 636, 'path': 'sentiment_analysis_IMDB_reviews/notebooks/text_processing.ipynb', 'repo_name': 'OBorovec/ds-experiments', 'size': '6417'}