<jupyter_start><jupyter_text>try 1  import<jupyter_code>from implicit.evaluation import  *
from implicit.als import AlternatingLeastSquares as ALS
from implicit.bpr import BayesianPersonalizedRanking as BPR
import numpy as np
import os
os.environ['KMP_DUPLICATE_LIB_OK']='True'
from sklearn.utils import shuffle
from scipy.sparse import *
# import rec_util
pd.options.display.float_format = '{:.2f}'.format
import pandas as pd<jupyter_output><empty_output><jupyter_text>파일 로드<jupyter_code>raw_tr = pd.read_json("./raw_data/train.json")
raw_te = pd.read_json("./raw_data/val.json")<jupyter_output><empty_output><jupyter_text>태그별 빈도 확인<jupyter_code>ret = []
for tag in raw_tr.tags.tolist():
    ret += tag
from collections import Counter
r = dict(Counter(ret))
r = sorted(r.items(), key=lambda x: -x[1])
r[:5]
top_tags = [x[0] for x in r[:1000]]<jupyter_output><empty_output><jupyter_text>태그와 노래 목록 만으로 나머지 예측하기<jupyter_code>train_songs = raw_tr['songs'].tolist()
test_songs = raw_te['songs'].apply(lambda x : [float(i)for i in x]).tolist()
train_tags = raw_tr['tags'].tolist()
test_tags = raw_te['tags'].tolist()
test_ids = raw_te['id'].tolist()<jupyter_output><empty_output><jupyter_text>train 데이터 list to list 로 만들기<jupyter_code>from itertools import groupby
tr = []
iid_to_idx = {}
idx = 0

for i, l in enumerate(train_songs):
    view = l
    for item_id in view:
        if item_id not in iid_to_idx:
            iid_to_idx[item_id] = idx
            idx += 1
    view = [iid_to_idx[x] for x in view]
    tr.append(view)
idx = 0
n_items = 615142
tag_to_idx = {} 
for i, tags in enumerate(train_tags):
    for tag in tags:
        if tag not in tag_to_idx:
            tag_to_idx[tag] = n_items + idx
            idx += 1
    tr[i].extend([tag_to_idx[x] for x in tags])
n_items = len(iid_to_idx)
n_tags = len(tag_to_idx)<jupyter_output><empty_output><jupyter_text>test 데이터 list to list 로 만들기<jupyter_code>from s import groupby
te = []

idx = 0
for i, l in enumerate(test_songs):
    view = l
    ret = [] 
    for item_id in view:
        if item_id not in iid_to_idx:
            continue
        ret.append(iid_to_idx[item_id])
    te.append(ret)
idx = 0
for i, tags in enumerate(test_tags):
    ret = []
    for tag in tags:
        if tag not in tag_to_idx:
            continue
        ret.append(tag)
    te[i].extend([tag_to_idx[x] for x in ret])<jupyter_output><empty_output><jupyter_text>데이터 쓰까<jupyter_code>tr = shuffle(tr)<jupyter_output><empty_output><jupyter_text>노래 및 태그 인덱싱 하기<jupyter_code>idx_to_iid = {x:y for(y,x) in iid_to_idx.items()}
idx_to_tag = {(x - n_items):y for(y,x) in tag_to_idx.items()}<jupyter_output><empty_output><jupyter_text>sparse matrix- 방법에 대해 고민 해봐야 함<jupyter_code>def boolean_indexing(v):
    lens = np.array([len(item) for item in v])
    mask = lens[:,None] > np.arange(lens.max())
    out = np.zeros(mask.shape,dtype=int)
    out[mask] = np.concatenate(v)
    return out
tr_array = boolean_indexing(tr)
te_array = boolean_indexing(te)
tr_array = np.zeros([len(tr),len(max(tr,key = lambda x: len(x)))])
for i,j in enumerate(tr):
    tr_array[i][0:len(j)] = j
# 노래 곡수 + 태그의 최대 숫자가 240인데 이렇게 나오면 안되고, 'n_tags + n_items'인 115071 x 644302인 어레이가 나와야 함
np.array([len (i) for i in tr]).max()
tr_array.astype(int)[0]
te_array.shape
n_tags + n_items<jupyter_output><empty_output><jupyter_text>binary matrix 만들기- https://stackoverflow.com/questions/50702988/creating-binary-matrix-of-each-unique-value-from-list-of-lists - train list to list<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer(tokenizer=lambda x: x, lowercase=False)
tr_array = cv.fit_transform(tr)

# To transform to dense matrix
tr_array.todense()<jupyter_output><empty_output><jupyter_text>- test list to list<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer(tokenizer=lambda x: x, lowercase=False)
te_array = cv.fit_transform(te)

# To transform to dense matrix
te_array.todense()<jupyter_output><empty_output><jupyter_text>모델링- 1. sparse 행렬 작업- 2. 두 행렬을 합치고- 3. 모델 fit 시키기<jupyter_code>tr_csr = csr_matrix(tr_array, (len(tr), n_tags + n_items))
te_csr = csr_matrix(te_array, (len(te), n_tags + n_items))
import scipy.sparse
r = scipy.sparse.vstack([te_csr, tr_csr])
r = csr_matrix(r)
als_model = ALS(factors=128, regularization=0.08)
als_model.fit(r.T * 15.0)<jupyter_output><empty_output><jupyter_text>노래 예측 모댈과, 태그 예측모댈에 user factor와 item factor를 분배<jupyter_code>item_model = ALS(use_gpu=False)
tag_model = ALS(use_gpu=False)
item_model.user_factors = als_model.user_factors
tag_model.user_factors = als_model.user_factors
item_model.item_factors = als_model.item_factors[:n_items]
tag_model.item_factors = als_model.item_factors[n_items:]
als_model.item_factors.shape
als_model.item_factors[n_items:]
item_rec_csr = tr_csr[:, :n_items]
tag_rec_csr = tr_csr[:, n_items:]<jupyter_output><empty_output><jupyter_text>recommend 과정<jupyter_code>item_ret = []
tag_ret = []
from tqdm.auto import tqdm
for u in tqdm(range(te_csr.shape[0])):
    item_rec = item_model.recommend(u, item_rec_csr, N=100)
    item_rec = [idx_to_iid[x[0]] for x in item_rec]
    tag_rec = tag_model.recommend(u, tag_rec_csr, N=100)
    tag_rec = [idx_to_tag[x[0]] for x in tag_rec if x[0] in idx_to_tag]
    item_ret.append(item_rec)
    tag_ret.append(tag_rec)
tag_model.item_factors
tag_model.item_factors<jupyter_output><empty_output><jupyter_text>최종 json파일 추출<jupyter_code>returnval = []
for _id, rec, tag_rec in zip(test_ids, item_ret, tag_ret):
    returnval.append({
        "id": _id,
        "songs": rec[:100],
        "tags": tag_rec[:10]
    })
import json
with open('ret.json', 'w', encoding='utf-8') as f:
    f.write(json.dumps(returnval, ensure_ascii=False))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12772963604852686, 'fraction_numerical': 0.012824956672443674, 'mean_word_length': 5.093980992608237, 'pattern_counts': {'":': 3, '<': 61, '<?xml version=': 0, '>': 62, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '115837', 'license': 'MIT', 'n_tokens_mistral': 2524, 'n_tokens_neox': 2564, 'n_words': 466, 'path': 'howard/playlist_recommendation/melon-recommandation-CMF.ipynb', 'repo_name': 'hmkim312/recommendation_project', 'size': '5770'}