<jupyter_start><jupyter_text>Cosine Similarity<jupyter_code>from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import math

stop_words = stopwords.words('english')

X = "I love horror movies movies movies".lower()
Y = "I also love horror movies".lower()

x_list = word_tokenize(X)
y_list = word_tokenize(Y)

features = sorted(list(set(x_list + y_list)))
features = [each for each in features if each not in stop_words]

l1 = [0] * len(features)
l2 = [0] * len(features)


# intially l1 & l2 are 0 now we will populate it.

for index,each in enumerate(features):
    if each in x_list:
        l1[index] = x_list.count(each)
    if each in y_list:
        l2[index] = y_list.count(each)

l1
l2
features
#Calculate the cosine similarity B/w l1 & l2
def cosine_sim(l1,l2):
    mul = [l1[i] * l2[i] for i in range (len(l1))]
    num = 0
    for i in range (0,len(mul)):
        num = num + mul[i]
    
    
    l1_sqr = [l1[i] * l1[i] for i in range (len(l1))]
    l1_sum = 0
    for i in range(0, len(l1_sqr)):
        l1_sum = l1_sum + l1_sqr[i]

    
    l2_sqr = [l2[i] * l2[i] for i in range (len(l2))]
    l2_sum = 0
    for i in range(0, len(l2_sqr)):
        l2_sum = l2_sum + l2_sqr[i]

    
    d1 = math.sqrt(l1_sum)
    d2 = math.sqrt(l2_sum)
    dnm = d1 * d2
    cos = num / dnm
    return cos
    

cosine_sim(l1,l2)
#by using function
from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer()
data = cv.fit_transform([X,Y])
data.toarray()
from sklearn.metrics.pairwise import cosine_similarity
cosine_similarity(data)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11885500933416304, 'fraction_numerical': 0.031113876789047916, 'mean_word_length': 3.334231805929919, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '44994', 'license': 'MIT', 'n_tokens_mistral': 671, 'n_tokens_neox': 655, 'n_words': 172, 'path': 'cosine_similarity.ipynb', 'repo_name': 'Shefali24M/NLP', 'size': '1607'}