<jupyter_start><jupyter_text>PreprocessingInspired from this [article](https://towardsdatascience.com/nlp-building-text-cleanup-and-preprocessing-pipeline-eba4095245a0).  Removing HTML tags<jupyter_code># imports
from bs4 import BeautifulSoup
# function to remove HTML tags
def remove_html_tags(text):
    return BeautifulSoup(text, 'html.parser').get_text()
# call function
remove_html_tags(" <html> \
 <h1>Article Heading</h1> \
 <p>First sentence of some important article. And another one. And then the last one</p></html>")<jupyter_output><empty_output><jupyter_text>Removing Accented Characters<jupyter_code># imports
import unicodedata
# function to remove accented characters
def remove_accented_chars(text):
    new_text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')
    return new_text
# call function
remove_accented_chars('Sómě Áccěntěd těxt. Some words such as résumé, café, prótest, divorcé, coördinate, exposé, latté.')<jupyter_output><empty_output><jupyter_text>Expanding Contractions<jupyter_code># imports
import contractions

contractions.fix("Y’all i’d contractions you’re expanded don’t think.")<jupyter_output><empty_output><jupyter_text>Removing Special Characters<jupyter_code># imports
import re
# function to remove special characters
def remove_special_characters(text):
    # define the pattern to keep
    pat = r'[^a-zA-z0-9.,!?/:;\"\'\s]' 
    return re.sub(pat, '', text)
 
# call function
remove_special_characters("007 Not sure@ if this % was #fun! 558923 What do# you think** of it.? $500USD!")<jupyter_output><empty_output><jupyter_text>Removing Numbers<jupyter_code># imports
import re
# function to remove numbers
def remove_numbers(text):
    # define the pattern to keep
    pattern = r'[^a-zA-z.,!?/:;\"\'\s]' 
    return re.sub(pattern, '', text)
 
# call function
remove_numbers("007 Not sure@ if this % was #fun! 558923 What do# you think** of it.? $500USD!")<jupyter_output><empty_output><jupyter_text>Removing Punctuation<jupyter_code># imports
import string
# function to remove punctuation
def remove_punctuation(text):
    text = ''.join([c for c in text if c not in string.punctuation])
    return text
# call function
remove_punctuation('Article: @First sentence of some, {important} article having lot of ~ punctuations. And another one;!')<jupyter_output><empty_output><jupyter_text>Stemming<jupyter_code># imports
import nltk
# function for stemming
def get_stem(text):
    stemmer = nltk.porter.PorterStemmer()
    text = ' '.join([stemmer.stem(word) for word in text.split()])
    return text
# call function
get_stem("we are eating and swimming ; we have been eating and swimming ; he eats and swims ; he ate and swam ")<jupyter_output><empty_output><jupyter_text>Lemmatization<jupyter_code># imports
import spacy
nlp = spacy.load("en_core_web_sm")
# function to remove special characters
def get_lem(text):
    text = nlp(text)
    text = ' '.join([word.lemma_ if word.lemma_ != '-PRON-' else word.text for word in text])
    return text
# call function
get_lem("we are eating and swimming ; we have been eating and swimming ; he eats and swims ; he ate and swam ")<jupyter_output><empty_output><jupyter_text>Removing Stopwords<jupyter_code># imports
import nltk
from nltk.tokenize import ToktokTokenizer
tokenizer = ToktokTokenizer()
stopword_list = nltk.corpus.stopwords.words('english')
# custom: removing words from list
stopword_list.remove('not')
# function to remove stopwords
def remove_stopwords(text):
    # convert sentence into token of words
    tokens = tokenizer.tokenize(text)
    tokens = [token.strip() for token in tokens]
    # check in lowercase 
    t = [token for token in tokens if token.lower() not in stopword_list]
    text = ' '.join(t)    
    return text
# call function
remove_stopwords("i am myself you the stopwords list and this article is not should removed")<jupyter_output><empty_output><jupyter_text>Removing extra whitespaces and tabs<jupyter_code># imports
import re
# function to remove special characters
def remove_extra_whitespace_tabs(text):
    #pattern = r'^\s+$|\s+$'
    pattern = r'^\s*|\s\s*'
    return re.sub(pattern, ' ', text).strip()
# call function
remove_extra_whitespace_tabs('  This web line  has \t some extra  \t   tabs and whitespaces  ')<jupyter_output><empty_output><jupyter_text>Lowercase<jupyter_code># function to remove special characters
def to_lowercase(text):
    return text.lower()
# call function
to_lowercase('ConVert THIS string to LOWER cASe.')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1111111111111111, 'fraction_numerical': 0.00834431269213878, 'mean_word_length': 5.5728715728715725, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 51, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '591568', 'license': 'MIT', 'n_tokens_mistral': 1553, 'n_tokens_neox': 1492, 'n_words': 417, 'path': 'notebooks/1.0-sb-preprocessing-pipeline.ipynb', 'repo_name': 'sboomi/cp1', 'size': '4554'}