<jupyter_start><jupyter_text>Import libraries and dataset<jupyter_code>import pandas as pd
import nltk
import re
from sklearn.feature_extraction.text import CountVectorizer
import requests
paragraf = requests.get('https://github.com/rasyidev/well-known-datasets/raw/main/rendang.txt').text
paragraf<jupyter_output><empty_output><jupyter_text>Tokenize each paragraph<jupyter_code>sentences = nltk.sent_tokenize(paragraf)
sentences
words_table = []

for i in range(len(sentences)):
    words = re.sub('[^a-zA-Z]', ' ', sentences[i])
    words = words.lower()
    words = words.split()
    words = ' '.join(words)
    words_table.append(words)
words_table<jupyter_output><empty_output><jupyter_text>CountVectorizer<jupyter_code>cv = CountVectorizer()
X = cv.fit_transform(words_table).toarray()
X<jupyter_output><empty_output><jupyter_text>Cosine Similarity<jupyter_code>from scipy import spatial
List1 = X[0]
List2 = X[1]
List3 = X[2]

result = 1 - spatial.distance.cosine(List1, List2, List3)
print(result)<jupyter_output>0.408248290463863<jupyter_text>Bag of Words<jupyter_code>index = ['Kalimat 1',
         'Kalimat 2',
         'Kalimat 3',
         'Kalimat 4',
         'Kalimat 5',
         'Kalimat 6',
         'Kalimat 7',
         'Kalimat 8',
         'Kalimat 9',
         'Kalimat 10',
         'Kalimat 11',
         'Kalimat 12',
         'Kalimat 13',
         'Kalimat 14',
         'Kalimat 15',
         'Kalimat 16',
         'Kalimat 17',
         'Kalimat 18',
         'Kalimat 19',
         'Kalimat 20']
words = cv.get_feature_names()
BoW = pd.DataFrame(X, columns=words, index=index)
pd.set_option("display.max_columns", None)
BoW<jupyter_output>C:\Python39\lib\site-packages\sklearn\utils\deprecation.py:87: FutureWarning: Function get_feature_names is deprecated; get_feature_names is deprecated in 1.0 and will be removed in 1.2. Please use get_feature_names_out instead.
  warnings.warn(msg, category=FutureWarning)<jupyter_text>Visualization with WordCloud<jupyter_code>from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator
import matplotlib.pyplot as plt

text = str(words).replace("'","")

# Create and generate a word cloud image:
wordcloud = WordCloud(max_font_size=50,
                      max_words=100,
                      background_color="white",
                      prefer_horizontal=0.80).generate(text)
plt.figure(figsize=(10,50))
plt.imshow(wordcloud, interpolation="bilinear")
plt.title("WordCloud", fontsize=30)
plt.axis("off")
plt.show()<jupyter_output><empty_output><jupyter_text>Count the word occurrences<jupyter_code>BoW_table = pd.DataFrame(BoW.sum())
pd.set_option("display.max_rows", None)
BoW_table
BoW.sum().sum()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12380250552689757, 'fraction_numerical': 0.029108327192336036, 'mean_word_length': 4.2011494252873565, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '235071', 'license': 'MIT', 'n_tokens_mistral': 1026, 'n_tokens_neox': 954, 'n_words': 173, 'path': '16 - Bag of Words/Bag_Of_Words.ipynb', 'repo_name': 'algonacci/Projects', 'size': '2714'}