<jupyter_start><jupyter_text>Data 가지고 오기<jupyter_code>genre_gn_all = pd.read_json('./Data/genre_gn_all.json', typ = 'series', encoding='utf-8')
song_meta = pd.read_json('./Data/song_meta.json', encoding='utf-8')
val= pd.read_json('./Data/val.json', encoding='utf-8', typ = 'frame')
train = pd.read_json('./Data/train.json', encoding='utf-8')
test = pd.read_json('./Data/test.json', encoding='utf-8')<jupyter_output><empty_output><jupyter_text>데이터 형태 분석<jupyter_code># 장르코드 : gnr_code, 장르명 : gnr_name
genre_gn_all = pd.DataFrame(genre_gn_all, columns = ['gnr_name']).reset_index().rename(columns = {'index' : 'gnr_code'})
genre_gn_all.head(3)
song_meta.head(3)
val.head(3)
train.head(3)
test.head(3)<jupyter_output><empty_output><jupyter_text>Info 보기<jupyter_code>song_meta.info()
train.info()
# 곡 아이디(id)와 대분류 장르코드 리스트(song_gn_gnr_basket) 추출
song_gnr_map = song_meta.loc[:, ['id', 'song_gn_gnr_basket']]

# unnest song_gn_gnr_basket
song_gnr_map_unnest = np.dstack(
    (
        np.repeat(song_gnr_map.id.values, list(map(len, song_gnr_map.song_gn_gnr_basket))), 
        np.concatenate(song_gnr_map.song_gn_gnr_basket.values)
    )
)

# unnested 데이터프레임 생성 : song_gnr_map
song_gnr_map = pd.DataFrame(data = song_gnr_map_unnest[0], columns = song_gnr_map.columns)
song_gnr_map['id'] = song_gnr_map['id'].astype(str)
song_gnr_map.rename(columns = {'id' : 'song_id', 'song_gn_gnr_basket' : 'gnr_code'}, inplace = True)

# unnest 객체 제거
del song_gnr_map_unnest
song_gnr_map.head(3)<jupyter_output><empty_output><jupyter_text>데이터 전처리  장르 처리<jupyter_code># 상세 장르 코드
song_meta.T[6:-2].T.head(3)
encoder.fit(genre_gn_all.gnr_code)
tmp = pd.DataFrame(encoder.transform(genre_gn_all.gnr_code))
tmp = tmp.rename(columns={
    '0': 'gnr_code'
})

gnr_code = genre_gn_all['gnr_code']

genre_gn_all['gnrOneHot'] = tmp

genre_gn_all.head(3)<jupyter_output><empty_output><jupyter_text>Tag 처리<jupyter_code>from sklearn import preprocessing

# 플레이리스트 아이디(id)와 매핑된 태그(tags) 추출
plylst_tag_map = train[['id', 'tags']]

# unnest tags
plylst_tag_map_unnest = np.dstack(
    (
        np.repeat(plylst_tag_map.id.values, list(map(len, plylst_tag_map.tags))), 
        np.concatenate(plylst_tag_map.tags.values)
    )
)

# unnested 데이터프레임 생성 : plylst_tag_map
plylst_tag_map = pd.DataFrame(data = plylst_tag_map_unnest[0], columns = plylst_tag_map.columns)
plylst_tag_map['id'] = plylst_tag_map['id'].astype(str)

# unnest 객체 제거
del plylst_tag_map_unnest

tags = plylst_tag_map.tags

tags.T.tail(3)


encoder.fit(tags)
tmp = pd.DataFrame(encoder.transform(plylst_tag_map.tags))

plylst_tag_map['tagEncoding'] = tmp

del tmp

plylst_tag_map.head(3)
df = pd.DataFrame({"tags":plylst_tag_map.tags, "tagEncoding":plylst_tag_map.tagEncoding})

name_idx_map = { r['tags']:r['tagEncoding'] for _, r in df.iterrows() }

data = pd.Series(train.tags)

train['tagsEncoding'] = data.apply(lambda e: [ name_idx_map[k] for k in e ])

del data, name_idx_map
train.head()
train['tags'] = train['tags'].fillna('')
train.head(2)
from sklearn.feature_extraction.text import CountVectorizer

def dummy(doc):
    return doc

tfidf = CountVectorizer(
    tokenizer=dummy,
    preprocessor=dummy,
) 
tfidf_matrix = tfidf.fit_transform(train.tags)
# overview에 대해서 tf-idf 수행
print(tfidf_matrix.shape)
tfidf.get_feature_names()
# from sklearn.metrics.pairwise import linear_kernel
# cosine_sim = linear_kernel(tfidf_matrix, tfidf_matrix)

# indices = pd.Series(data.index, index=data['tags']).drop_duplicates()
# print(indices.head())
# from sklearn.metrics.pairwise import cosine_similarity

# genre_sim = cosine_similarity(tfidf_matrix, tfidf_matrix, batch_size = 40)
# print(genre_sim.shape)
# print(genre_sim[:2])
# tmp = np.array(train.tags)
# train.songs.shape

# from tensorflow.keras.layers import Input, Dense
# from tensorflow.keras.models import Model

# # MNIST input 28 rows * 28 columns = 784 pixels
# input_img = Input(shape=(7,))
# # encoder
# encoder1 = Dense(128, activation='sigmoid')(input_img)
# encoder2 = Dense(3, activation='sigmoid')(encoder1)
# # decoder
# decoder1 = Dense(128, activation='sigmoid')(encoder2)
# decoder2 = Dense(7, activation='sigmoid')(decoder1)

# # this model maps an input to its reconstruction
# autoencoder = Model(inputs=input_img, outputs=decoder2)

# autoencoder.compile(optimizer='adam', loss='binary_crossentropy')
# autoencoder.fit(train, train,
#                 epochs=5,
#                 batch_size=32,
#                 shuffle=True,
#                 validation_data=(test.tags, test.tags))
def pearsonR(s1, s2):
    s1_c = s1 - s1.mean()
    se_c = s2 - s2.mean()
    return np.sum(s1_c * s2_c) / np.sqrt(np.sum(s1_c ** 2) * np.sum(s2_c ** 2))

# def rec

def cos_sim(A, B):
       return np.dot(A, B)/(norm(A)*norm(B))
# print(cos_sim(genre_mat, genre_mat))
import keras

keras.losses.CosineSimilarity(
    axis=-1, reduction=losses_utils.ReductionV2.AUTO, name='cosine_similarity'
)

cosine_loss = tf.keras.losses.CosineSimilarity(axis=1)

model = tf.keras.Model(tfidf_matrix, tfidf_matrix)
model.compile('sgd', loss=tf.keras.losses.CosineSimilarity(axis=1))<jupyter_output>Using TensorFlow backend.<jupyter_text>학습<jupyter_code>df = pd.DataFrame({"name":["발", "팝", "팔"], "onehot":[0,1,2]})
df
name_idx_map = { r['name']:r['onehot'] for _, r in df.iterrows() }
name_idx_map
data = pd.Series([ ['발'], ['팝', '팔']])
data.dtype
data.apply(lambda e: [ name_idx_map[k] for k in e ])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1535338623362244, 'fraction_numerical': 0.013102048348403764, 'mean_word_length': 5.666666666666667, 'pattern_counts': {'":': 4, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '656358', 'license': 'MIT', 'n_tokens_mistral': 2369, 'n_tokens_neox': 2403, 'n_words': 339, 'path': 'train.ipynb', 'repo_name': 'brainer3220/Melon-Playlist-Continuation', 'size': '5419'}