<jupyter_start><jupyter_text>EDA<jupyter_code>%load_ext autoreload
%autoreload 2
import sys
sys.path.append("../src")
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import scipy
from collections import Counter<jupyter_output><empty_output><jupyter_text>Data Preparation<jupyter_code>ds_links = pd.read_csv("../ml-latest-small/links.csv")
ds_movies = pd.read_csv("../ml-latest-small/movies.csv")
ds_ratings = pd.read_csv("../ml-latest-small/ratings.csv")
ds_tags = pd.read_csv("../ml-latest-small/tags.csv")<jupyter_output><empty_output><jupyter_text>Ratings<jupyter_code>ds_movies
ratings_mean_count = ds_ratings.groupby("movieId").agg({"rating": ["mean", "count"]})
ratings_mean_count.columns = ["ratings_mean", "ratings_count"]
ratings_mean_count<jupyter_output><empty_output><jupyter_text>User-Item Matrix<jupyter_code>def create_user_item_matrix(ratings) -> pd.DataFrame:
  mat = ratings.pivot(index="userId", columns="movieId", values="rating")
  mat[~mat.isna()] = 1
  mat.fillna(0, inplace=True)
  return mat

def create_user_item_ranking_matrix(ratings) -> pd.DataFrame:
    return ratings.pivot(index="userId", columns="movieId", values="rating")

user_item_matrix = create_user_item_ranking_matrix(ds_ratings)
user_item_matrix.head()<jupyter_output><empty_output><jupyter_text>유저마다 시간별로 rating 매긴걸 정렬한 다음일정 비율로 나눠서 이전에 매긴 rating을 바탕으로 이후에 매긴 rating을 예측할 수 있는모델을 만들고자 함<jupyter_code>def create_user_item_matrix_train(ds_ratings, train_size=0.5):
    users = ds_ratings.userId.unique()
    train_sets = []
    for user in users:
        ratings = ds_ratings[ds_ratings.userId == user].sort_values("timestamp")
        q_value = ratings.timestamp.quantile(q=train_size, interpolation='nearest')
        train_set = ratings[ratings.timestamp <= q_value]
        train_sets.append(train_set)

    train_set = pd.concat(train_sets, axis=0)
    return train_set

train_set = create_user_item_matrix_train(ds_ratings, 0.5)
train_set = create_user_item_ranking_matrix(train_set)
train_set
train_set.to_csv("../data/user_item_matrix_X_0.5.csv")
create_user_item_matrix(ds_ratings).to_csv("../data/user_item_matrix_Y_0.5.csv")<jupyter_output><empty_output><jupyter_text>Tag 정보 EDA<jupyter_code>tag_counts = ds_tags.tag.value_counts()
tag_counts[tag_counts >= 5]<jupyter_output><empty_output><jupyter_text>Movie EDA<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
# ds_movies.genres.str.split("|")
cv = CountVectorizer()
genres = cv.fit_transform(ds_movies.genres)
genres = pd.DataFrame(
    genres.toarray(),
    columns=list(sorted(cv.vocabulary_.keys(), key=lambda x: cv.vocabulary_[x]))
)
genres<jupyter_output><empty_output><jupyter_text>title 에서 연도를 지울 경우 중복되는게 있는지 확인해봤고, 282개 있음<jupyter_code>titles = ds_movies.title.str[:-7]
titles.value_counts()
ds_movies[ds_movies.title.str.contains("Hamlet")]<jupyter_output><empty_output><jupyter_text>학습에 필요한 결과물 저장<jupyter_code>movie_genre_matrix = pd.concat(
    [ds_movies.drop(columns="genres"), genres],
    axis=1
).drop(columns="movieId")
movie_genre_matrix.index = ds_movies.movieId
movie_genre_matrix = pd.concat([movie_genre_matrix, ratings_mean_count], axis=1)
movie_genre_matrix["years"] = movie_genre_matrix.title.str.strip().str[-5:-1]
movie_genre_matrix.years = movie_genre_matrix.years.apply(lambda x: int(x) if x.isdigit() else 0)

movie_genre_matrix.to_csv("../data/movies.csv")
movie_genre_matrix
movie_genre_matrix.columns[1:25]<jupyter_output><empty_output><jupyter_text>영화 점보와 평점 정보를 pandas를 이용해서 불러옵니다<jupyter_code>ds_movies = pd.read_csv("../ml-latest-small/movies.csv", index_col=0)
ds_ratings = pd.read_csv("../ml-latest-small/ratings.csv")<jupyter_output><empty_output><jupyter_text>영화 평점과 평가 수를 집계한 후 영화 제목을 데이터프레임에 추가합니다.<jupyter_code>movie_ratings = ds_ratings.groupby("movieId") \
                          .agg({"rating": ["count", "mean"]})
movie_ratings["title"] = ds_movies.title
movie_ratings<jupyter_output><empty_output><jupyter_text>평점이 50개 이상한 영화 중 가장 평점이 좋은 영화 10개를 가져옵니다.평점이 같다면 평가 수가 많은 영화 순으로 정렬합니다.<jupyter_code>movie_ratings[movie_ratings[("rating", "count")] >= 50] \
             .sort_values([("rating", "mean"), ("rating", "count")], ascending=[False, False]) \
             .head(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13618313478159308, 'fraction_numerical': 0.007942069609904227, 'mean_word_length': 6.660107334525939, 'pattern_counts': {'":': 2, '<': 50, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '5868', 'license': 'MIT', 'n_tokens_mistral': 1842, 'n_tokens_neox': 1901, 'n_words': 232, 'path': 'notebooks/EDA.ipynb', 'repo_name': 'HeegyuKim/RecSys-MovieLens100k', 'size': '4281'}