<jupyter_start><jupyter_text>Stop Words  nltk<jupyter_code>import nltk

from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
nltk.download('stopwords')
nltk.download('punkt')
def remove_stopwords_nltk(text):
  stop_words_english = set(stopwords.words('english'))
  tokens = word_tokenize(text)

  filtered_text = [word for word in tokens if not word in stop_words_english]

  return filtered_text
input_text = "SpaceX is an American aerospace manufacturer, space transportation services and communications company headquartered in Hawthorne, California"

remove_stopwords_nltk(input_text)<jupyter_output><empty_output><jupyter_text>spacy<jupyter_code>import spacy
sp = spacy.load('en_core_web_sm')
def remove_stopwords_spacy(text):
  stop_words_english = sp.Defaults.stop_words
  tokens = word_tokenize(text)

  filtered_text = [word for word in tokens if not word in stop_words_english]

  return filtered_text
input_text = "SpaceX is an American aerospace manufacturer, space transportation services and communications company headquartered in Hawthorne, California"

remove_stopwords_spacy(input_text)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.0683982683982684, 'fraction_numerical': 0.0, 'mean_word_length': 7.757575757575758, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '432570', 'license': 'MIT', 'n_tokens_mistral': 380, 'n_tokens_neox': 367, 'n_words': 91, 'path': 'Data-Preprocessing/StopWords.ipynb', 'repo_name': 'guru9573/NLP', 'size': '1155'}