<jupyter_start><jupyter_text>*        바나나     사과     저는     좋아요* 문서1     0          1        1        1* 문서2     1          0        1        1* 문서3     2          0        2        2* 참고 : https://wikidocs.net/24603<jupyter_code>from numpy import dot
from numpy.linalg import norm
import numpy as np
def cos_sim(A, B):
       return dot(A, B)/(norm(A)*norm(B))
doc1 = np.array([0,1,1,1])
doc2 = np.array([1,0,1,1])
doc3 = np.array([2,0,2,2])
# 인접행렬
cos_sim(doc1, doc2)
cos_sim(doc1, doc3)
cos_sim(doc2, doc3)
from sklearn.feature_extraction.text import TfidfVectorizer
corpus = [
    'you know I want your love',
    'I like you',
    'what should I do ',    
]
tfidf = TfidfVectorizer()
print(tfidf.fit_transform(corpus).toarray())
print(tfidf.vocabulary_)<jupyter_output>{'you': 7, 'know': 1, 'want': 5, 'your': 8, 'love': 3, 'like': 2, 'what': 6, 'should': 4, 'do': 0}<jupyter_text>Recommended with cos similarity dataset from https://www.kaggle.com/rounakbanik/the-movies-dataset/version/7?select=movies_metadata.csv<jupyter_code>import pandas as pd
df_data = pd.read_csv('./movies_metadata.csv')
df_data.head(3)
df_data.columns
data = df_data.head(10000)
data = data.dropna()
data.info()
from sklearn.feature_extraction.text import TfidfVectorizer
tfidf = TfidfVectorizer()
tfidf_matrix = tfidf.fit_transform(data['overview'])
tfidf_matrix.shape
len(tfidf.vocabulary_)
type(tfidf_matrix)
tfidf_matrix[4].toarray()
from sklearn.metrics.pairwise import linear_kernel    # 커널은 차원이라고 이해하면 됨 랭커도 비슷
cosin_sim = linear_kernel(tfidf_matrix, tfidf_matrix)
type(cosin_sim), cosin_sim.shape
cosin_sim[3]
indexes = pd.Series(data.index, index=data['title']).drop_duplicates()
indexes.head(5)
indexes['From <NAME>']
list(cosin_sim[69])
cosin_list = list(enumerate(cosin_sim[69]))    # movie index
cosin_list
cosin_index = sorted(cosin_list, key=lambda x:x[1], reverse=True)
cosin_index[1:11]   # top 10
# for i in cosin_index:
#   i[0]

movie_indexes = [i[0] for i in cosin_index[1:11]]
data['title'].iloc[movie_indexes]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1413894324853229, 'fraction_numerical': 0.03669275929549902, 'mean_word_length': 4.4388297872340425, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '655801', 'license': 'Apache-2.0', 'n_tokens_mistral': 879, 'n_tokens_neox': 875, 'n_words': 169, 'path': 'NLP/Cosine_Similarity.ipynb', 'repo_name': 'victoria2012/test_deeplearning', 'size': '2052'}