<jupyter_start><jupyter_text>Bag of WordsBag of Words based encoding or TF-IDF vector is a frequentist based approach to NLP applications.<jupyter_code>import urllib.request

url = "https://www.gutenberg.org/files/829/829-0.txt" # gulliver's travels
#Alt
#url = 'https://www.gutenberg.org/files/2701/2701-0.txt' # <NAME>

file = urllib.request.urlopen(url)
text = [line.decode('utf-8') for line in file]
text = ''.join(text)<jupyter_output><empty_output><jupyter_text>Tokenize<jupyter_code>nltk.download('punkt')
from nltk import word_tokenize
tokens = word_tokenize(text)
import string
tokens = [word for word in tokens if word.isalpha()]
table = str.maketrans('', '', string.punctuation)
tokens = [w.translate(table) for w in tokens]
tokens = [word.lower() for word in tokens]<jupyter_output><empty_output><jupyter_text>Removing **stop-words** and **stemming**<jupyter_code>from nltk.corpus import stopwords
nltk.download('stopwords')
stop_words = set(stopwords.words('english'))
tokens = [w for w in tokens if not w in stop_words]

from nltk.stem.porter import PorterStemmer
porter = PorterStemmer()
tokens = [porter.stem(word) for word in tokens]
tokens[200:202]<jupyter_output>[nltk_data] Downloading package stopwords to
[nltk_data]     /home/yashroff/nltk_data...
[nltk_data]   Package stopwords is already up-to-date!<jupyter_text>**Understanding the vocabulary** * A vocabulary of a document represents all the words in that document and the frequency they appear.* `FreqDist` class<jupyter_code>from nltk.probability import FreqDist

word_counts = FreqDist(tokens)
word_counts<jupyter_output><empty_output><jupyter_text>**Scoring words with frequency**<jupyter_code>top = 100
vocabulary = word_counts.most_common(top)

vocabulary[:10]
voc_size = len(vocabulary)
doc_vector = np.zeros(voc_size)

word_vector = [(idx,word_counts[word[0]]) for idx, word in enumerate(vocabulary) if word[0] in word_counts.keys()] 
word_vector[10]
# Generating a model of Bag of Words

from nltk import sent_tokenize

docs = sent_tokenize(text)[703:706]
docs

from sklearn.feature_extraction.text import CountVectorizer

count_vectorizer=CountVectorizer(stop_words='english')
word_count_vector=count_vectorizer.fit_transform(docs)
word_count_vector.shape
word_count_vector.toarray()
count_vectorizer.get_feature_names()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11377245508982035, 'fraction_numerical': 0.016253207869974338, 'mean_word_length': 7.149825783972125, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '458798', 'license': 'MIT', 'n_tokens_mistral': 860, 'n_tokens_neox': 826, 'n_words': 186, 'path': 'part1_labs/01_NLP basics.ipynb', 'repo_name': 'ravi-ilango/odsc2020_nlp', 'size': '2341'}