<jupyter_start><jupyter_text>Imports<jupyter_code>import nltk
import numpy as np
import re<jupyter_output><empty_output><jupyter_text>Remove html tags<jupyter_code>from io import StringIO
from html.parser import HTMLParser

class HTMLStrip(HTMLParser):
    def __init__(self):
        super().__init__()
        self.reset()
        self.strict = False
        self.convert_charrefs= True
        self.text = StringIO()
    def handle_data(self, d):
        self.text.write(d)
    def get_data(self):
        return self.text.getvalue()

def strip_tags(html):
    htmlStripperObj = HTMLStrip()
    htmlStripperObj.feed(html)
    return htmlStripperObj.get_data()
strip_tags("<!DOCTYPE html><html><body><p>This is a paragraph.</p><p>This is a paragraph.</p><p>This is a paragraph.</p></body></html>")<jupyter_output><empty_output><jupyter_text>Removing extra spaces<jupyter_code>class WhitespaceCorrect():
    def __init__(self,text):
        self.text = text
    def get_data(self):
        return (' '.join(self.text.split()))

def strip_space(text):
    whitespace = WhitespaceCorrect(text)
    return whitespace.get_data()
strip_space("ths.     is nothing")<jupyter_output><empty_output><jupyter_text>remove special characters<jupyter_code>class SpecialChar():
    def __init__(self,text):
        self.text = text
        self.mod = ""
    def get_data(self):
        final_text = ""
        for str1 in self.text.split(" "):
            str1 = re.sub('[\W\_]','',str1)
            self.mod = self.mod+' '+str1
        return (' '.join(self.mod.split()))

def strip_space(text):
    spl_char = SpecialChar(text)
    return spl_char.get_data()
strip_space("ths.     is #$#&^&#$@||||123515abc def")<jupyter_output><empty_output><jupyter_text>Lowercase or uppercase<jupyter_code>class CaseUnify():
    def __init__(self,text,case='lower'):
        self.text = text
        self.case=case
    def get_data(self):
        if self.case=='upper':
            return (self.text.upper())
        else:
            return( self.text.lower())
        

def case_unify(text,case):
    case_char = CaseUnify(text,case)
    return case_char.get_data()
case_unify("thIS.     is #$#&^&#$@||||123515Abc def",'lower')<jupyter_output><empty_output><jupyter_text>remove custom stopwords<jupyter_code>class StopWords():
    def __init__(self,text,stop_list):
        self.text = text
        self.stop_list=stop_list
    def get_data(self):
        return(' '.join([i for i in self.text.split(" ") if i not in self.stop_list]))
        

def rem_stopwords(text,stop_list):
    rem_stop = StopWords(text,stop_list)
    return rem_stop.get_data()
rem_stopwords('this is hello world', ['is'])
from remove_extra_space import WhitespaceCorrect
text_data = "nb. sjbs.  "
test = WhitespaceCorrect(text_data)
test.get_data()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13715093672675857, 'fraction_numerical': 0.005655708731000354, 'mean_word_length': 3.716666666666667, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '50597', 'license': 'BSD-2-Clause', 'n_tokens_mistral': 983, 'n_tokens_neox': 970, 'n_words': 176, 'path': 'nlp_preprocess/_test_preprocess.ipynb', 'repo_name': 'nitinranjansharma/nlp_preprocess', 'size': '2829'}