<jupyter_start><jupyter_text>M2. Data preparation<jupyter_code>import numpy as np
import pickle
from c15ym.items_data import ItemsData
from c15ym.scores_data import ScoresData
from sklearn.metrics.pairwise import cosine_similarity
from timer.timer import Timer
timer = Timer()
processed_data_dir_path = './../processed_data/'
items_data = pickle.load(open(processed_data_dir_path + 'c15ym_dataset_1_items_data.pickle', 'rb'))
scores_data = pickle.load(open(processed_data_dir_path + 'c15ym_dataset_1_train_scores_data.pickle', 'rb'))
users_area = np.arange(40000)
users_area_size = users_area.shape[0]
# for quick access to users' ratings in ratings matrices
users_2_area_indices = users_area # it is useless with users_area == [0..19999] but if you choose different users area, you will probably need in users_2_area_indices
all_artists = np.copy(items_data._artists)
all_artists_size = all_artists.shape[0]
all_artists_n_tracks = np.array(list(map(lambda artist: items_data.get_artist_tracks(artist).shape[0], all_artists)), dtype=np.int32) 
all_artists_n_albums = np.array(list(map(lambda artist: items_data.get_artist_albums(artist).shape[0], all_artists)), dtype=np.int32)
# we are interested in artists with at least 1 related track or album
all_artists_flags = (all_artists_n_tracks + all_artists_n_albums) > 0
artists = all_artists[all_artists_flags]
artists_size = artists.shape[0]
artists_n_items = all_artists_n_tracks[all_artists_flags] + all_artists_n_albums[all_artists_flags]
# for quick access to artists' ratings in ratings matrices
artists_2_indices = np.full(shape=np.max(all_artists) + 1, fill_value=np.int32(-1), dtype=np.int32)
for i in np.arange(artists_size):
    artists_2_indices[artists[i]] = i
artists_ratings = np.full(shape=(artists_size, users_area_size), fill_value=np.float32(-1), dtype=np.float32)
artists_items_acc = np.zeros(shape=(artists_size, users_area_size), dtype=np.float32)
artists_items_c = np.zeros(shape=(artists_size, users_area_size), dtype=np.int32)
for users_area_i in np.arange(users_area_size):
    items, scores = scores_data.get_user_data(user=users_area[users_area_i])
    tracks_indices = np.flatnonzero(items_data.is_track(items))
    for track, score in zip(items[tracks_indices], scores[tracks_indices]):
        track_artist = items_data.get_track_artist(track)
        if track_artist != np.int32(-1):
            artist_i = artists_2_indices[track_artist]
            if artist_i != np.int32(-1):
                artists_items_acc[artist_i, users_area_i] += score
                artists_items_c[artist_i, users_area_i] += 1
    albums_indices = np.flatnonzero(items_data.is_album(items))
    for album, score in zip(items[albums_indices], scores[albums_indices]):
        album_artist = items_data.get_album_artist(album)
        if album_artist != np.int32(-1):
            artist_i = artists_2_indices[album_artist]
            if artist_i != np.int32(-1):
                artists_items_acc[artist_i, users_area_i] += score
                artists_items_c[artist_i, users_area_i] += 1
    artists_indices = np.flatnonzero(items_data.is_artist(items))
    for artist, score in zip(items[artists_indices], scores[artists_indices]):
        artist_i = artists_2_indices[artist]
        if artist_i != np.int32(-1):
            artists_ratings[artist_i, users_area_i] = score
def get_artists_matrix(artists_ratings, artists_items_acc, artists_items_c):
    def transfom_ratings(ratings):
        threshold_rating = np.float32(0.5)
        return np.array([-1, 1], dtype=np.float32)[np.int32(ratings >= threshold_rating)]
    are_artists_rated = artists_ratings != np.float32(-1)
    are_artists_rated_through_items = np.logical_and(np.logical_not(are_artists_rated), artists_items_c >= np.int32(1))    
    artists_matrix = np.zeros(shape=artists_ratings.shape, dtype=np.float32)
    artists_matrix[are_artists_rated] = transfom_ratings(artists_ratings[are_artists_rated])
    artists_matrix[are_artists_rated_through_items] = transfom_ratings(artists_items_acc[are_artists_rated_through_items] / artists_items_c[are_artists_rated_through_items])
    return artists_matrix
train_artists_matrix = get_artists_matrix(artists_ratings[:, :20000], artists_items_acc[:, :20000], artists_items_c[:, :20000])
artists_sim = cosine_similarity(train_artists_matrix)
results_acc = np.zeros(shape=(2, 3), dtype=np.int32)

timer.start()
for users_area_i in np.arange(20000, 30000):
    #
    artists_indices = np.flatnonzero(artists_ratings[:, users_area_i] != np.float32(-1))
    artists_indices = artists_indices[np.random.permutation(artists_indices.shape[0])]
    train_size = np.int32(np.round(np.float32(0.5) * artists_indices.shape[0]))
    #
    train_artists_indices = artists_indices[:train_size]
    train_user_artists_ratings = artists_ratings[train_artists_indices, users_area_i]
    train_user_artists_items_acc = artists_items_acc[train_artists_indices, users_area_i]
    train_user_artists_items_c = artists_items_c[train_artists_indices, users_area_i]
    #
    test_artists_indices = artists_indices[train_size:]
    test_user_artists_ratings = artists_ratings[test_artists_indices, users_area_i]
    #
    for test_artist_i, test_rating in zip(test_artists_indices, test_user_artists_ratings):
        #
        result_i = 0
        if test_rating >= np.float32(0.5):
            result_i = 1
        #
        pos = np.int32(0)
        neg = np.int32(0)
        #
        for train_rating_i, train_rating in zip(train_artists_indices, train_user_artists_ratings):
            if artists_sim[test_artist_i, train_rating_i] >= np.float32(0.2):
                if train_rating >= np.float32(0.5):
                    pos += 1
                else:
                    neg += 1
        #
        result_j = 2
        if neg > 0 and neg > pos:
            result_j = 0
        if pos > 0 and pos >= neg:
            result_j = 1
        #
        results_acc[result_i, result_j] += 1
print(timer.stop())
print(results_acc)
print(np.round(np.float32(results_acc[0, :]) / np.sum(results_acc[0, :]), 4))
print(np.round(np.float32(results_acc[1, :]) / np.sum(results_acc[1, :]), 4))

for users_area_i in np.arange(20000, 40000):
    artists_indices = np.flatnonzero(artists_ratings[:, users_area_i] != np.float32(-1))
    artists_indices = artists_indices[np.random.permutation(artists_indices.shape[0])]
    train_size = np.int32(np.round(np.float32(0.75) * artists_indices.shape[0]))
    # train
    train_artists_indices = artists_indices[:train_size]
    train_artists_ratings = artists_ratings[train_artists_indices, users_area_i]
    train_artists_likes = train_artists_ratings >= np.float32(0.5)
    train_artists_0_indices = train_artists_indices[np.logical_not(train_artists_likes)]
    train_artists_1_indices = train_artists_indices[train_artists_likes]
    # test
    test_artists_indices = artists_indices[train_size:]
    test_artists_ratings = artists_ratings[test_artists_indices, users_area_i]
    test_artists_likes = test_artists_ratings >= np.float32(0.5)
    test_artists_0_indices = test_artists_indices[np.logical_not(test_artists_likes)]
    test_artists_1_indices = test_artists_indices[test_artists_likes]
    # testing<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10243295548797346, 'fraction_numerical': 0.027923693668786286, 'mean_word_length': 4.7557677008751, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '510480', 'license': 'MIT', 'n_tokens_mistral': 2677, 'n_tokens_neox': 2576, 'n_words': 393, 'path': 'm2_data_preparation__.ipynb', 'repo_name': 'miloiloloo/diploma_2019_mrs', 'size': '7234'}