<jupyter_start><jupyter_text>Bag of Words can be build in three steps:1. Split and cleaning data ( preprocessing data )2. Create the vocab, make a list of words in the document.2. Create a document vector and score the words in this document using a fixed length document representation2. The score can be by:        * Count        * TF-IDF        * Frequency        * binary<jupyter_code>text = '''It was the best of times,
it was the age of wisdom,
it's the worst of times,
it was a beautiful mess'''
tokens = text.split()
print(tokens)
from collections import Counter

Vocab = Counter()  
Vocab.update(tokens) # counter and update the number of words in the vocabulary
Vocab = set(Vocab)
print(Vocab)
print(len(Vocab))
from keras.preprocessing.text import Tokenizer

tokenizer = Tokenizer()
tokenizer.fit_on_texts(Vocab)


text1 = ['it was the only thing to do']
modes = ['count', 'tfidf', 'binary', 'freq'] # the four score methods used in bag of words

for mode in modes:
    text_vec = tokenizer.texts_to_matrix(text1, mode = mode)   # create the document vector
    print(f'mode: {mode} ')
    print(text_vec)
    print()
    
# For using a bag of words if n greater than 1:

from itertools import combinations

vocab2 = Counter()

vocab2.update(Counter(combinations(tokens, 2)))  # 2-gram 

print(set(vocab2))<jupyter_output>{('of', 'worst'), ('wisdom,', 'beautiful'), ("it's", 'the'), ('worst', 'times,'), ('times,', 'wisdom,'), ("it's", 'times,'), ('it', 'the'), ('age', 'was'), ('times,', 'beautiful'), ('best', 'times,'), ('age', 'a'), ('of', 'a'), ('worst', 'mess'), ('of', 'was'), ('best', 'the'), ('of', 'age'), ('it', 'beautiful'), ('times,', "it's"), ('was', 'beautiful'), ('was', 'best'), ('age', 'worst'), ('wisdom,', 'times,'), ('worst', 'beautiful'), ('worst', 'of'), ('the', 'times,'), ('best', 'age'), ('was', "it's"), ('the', 'worst'), ('was', 'mess'), ('It', 'times,'), ('it', 'mess'), ('of', 'the'), ('worst', 'it'), ('wisdom,', 'of'), ('times,', 'times,'), ('It', 'wisdom,'), ('of', 'beautiful'), ('wisdom,', 'worst'), ('the', 'wisdom,'), ('best', 'mess'), ('the', 'of'), ('it', 'age'), ('of', 'it'), ('the', 'a'), ('wisdom,', 'mess'), ('it', 'of'), ('best', 'worst'), ('the', 'age'), ('wisdom,', 'a'), ('best', 'of'), ('best', 'was'), ('it', 'worst'), ("it's", 'beautiful'), ('best', 'it'), ('of', 'times,[...]
{'fraction_non_alphanumeric': 0.25278015397775877, 'fraction_numerical': 0.0051325919589392645, 'mean_word_length': 5.220744680851064, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '111358', 'license': 'MIT', 'n_tokens_mistral': 845, 'n_tokens_neox': 811, 'n_words': 274, 'path': 'Sentimental Analysis/.ipynb_checkpoints/BagOfWords-checkpoint.ipynb', 'repo_name': 'rodrigoamorimml/NaturalLanguageProcessing', 'size': '2338'}