<jupyter_start><jupyter_text>1. Remove_Stopwords<jupyter_code>from gensim.parsing.preprocessing import remove_stopwords
mydata_stopremoved = remove_stopwords(mydata)
mydata_stopremoved
len(mydata_stopremoved)<jupyter_output><empty_output><jupyter_text>2. Preprocess_string<jupyter_code>from gensim.parsing.preprocessing import preprocess_string
preprocess_string("<i>Hel 9lo</i> <b>Wo9 rld</b>! Th3     weather_is really g00d today, isn't it?")
mydata_process_string = preprocess_string(mydata)
mydata_process_string
len(mydata_process_string)
from gensim.parsing.preprocessing import strip_tags, strip_punctuation
s = "<i>Hel 9lo</i> <b>Wo9 rld</b>! Th3     weather_is really g00d today, isn't it?"
CUSTOM_FILTERS = [lambda x: x.lower(), strip_tags, strip_punctuation]
preprocess_string(s, CUSTOM_FILTERS)<jupyter_output><empty_output><jupyter_text>3. Preprocess_documents<jupyter_code>from gensim.parsing.preprocessing import preprocess_documents
preprocess_documents(["<i>Hel 9lo</i> <b>Wo9 rld</b>!", "Th3     weather_is really g00d today, isn't it?"])
mydata_preprocess_documents = mydata.split('  ')
mydata_preprocess_documents
preprocess_documents(mydata_preprocess_documents)<jupyter_output><empty_output><jupyter_text>4. Split_Alphanum<jupyter_code>from gensim.parsing.preprocessing import split_alphanum
split_alphanum("24.0hours7 days365 a1b2c3")
#If the number is adjacent to text then space comes in between, not applicable for '.' and '-'
split_alphanum(mydata)<jupyter_output><empty_output><jupyter_text>5. Stem_Text<jupyter_code>from gensim.parsing.preprocessing import stem_text
stem_text("While it is quite useful to be able to search a large collection of documents almost instantly.")
stem_text(mydata)<jupyter_output><empty_output><jupyter_text>6. Strip_Multiple_Whitespaces<jupyter_code>from gensim.parsing.preprocessing import strip_multiple_whitespaces
strip_multiple_whitespaces("salut" + '\r' + " les" + '\n' + "         loulous!")

strip_multiple_whitespaces(mydata)<jupyter_output><empty_output><jupyter_text>7. Strip_Non_Alphanum<jupyter_code>from gensim.parsing.preprocessing import strip_non_alphanum
strip_non_alphanum("if-you#can%read$this&then@this#method^works")
strip_non_alphanum(mydata)<jupyter_output><empty_output><jupyter_text>8. Strip_Numeric<jupyter_code>from gensim.parsing.preprocessing import strip_numeric
strip_numeric("0text24gensim365test")
strip_numeric(mydata)<jupyter_output><empty_output><jupyter_text>9. Strip_Punctuation<jupyter_code>from gensim.parsing.preprocessing import strip_punctuation
strip_punctuation("A semicolon is a stronger break than a comma, but not as much as a full stop!")
strip_punctuation(mydata)<jupyter_output><empty_output><jupyter_text>10. Strip_Short<jupyter_code>from gensim.parsing.preprocessing import strip_short
strip_short("salut les amis du 59")
strip_short(mydata)<jupyter_output><empty_output><jupyter_text>11. Strip_Tags<jupyter_code>from gensim.parsing.preprocessing import strip_tags
strip_tags("<i>Hello</i> <b>World</b>!")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10039370078740158, 'fraction_numerical': 0.015091863517060367, 'mean_word_length': 10.462406015037594, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '223644', 'license': 'MIT', 'n_tokens_mistral': 1147, 'n_tokens_neox': 1101, 'n_words': 178, 'path': 'Gensim/1_Gensim_Preprocessing_Methods.ipynb', 'repo_name': 'sudevansujit/7_NLP', 'size': '3048'}