<jupyter_start><jupyter_text>Genre Prediction Based on Music Metadata and Calculated Features<jupyter_code>genres_info = pd.read_csv("data/fma/genres.csv", index_col = "genre_id")
genres_info
genres_info.sort_values("#tracks", ascending = False)
top_level_genre_ids = genres_info.top_level.unique()
top_level_genres = genres_info.loc[top_level_genre_ids]
top_level_genres = top_level_genres.sort_values("#tracks", ascending = True)[["title", "#tracks"]]
len(top_level_genres)
plt.figure(figsize = (10, 8))
plt.barh(top_level_genres["title"], top_level_genres["#tracks"])

plt.xlabel("Number of tracks")
plt.title("Number of tracks per top genre")

plt.show()
tracks_info = pd.read_csv("data/fma/tracks.csv", header = [0, 1])
tracks_info = tracks_info.loc[1:]
tracks_info
tracks_info.columns
tracks_info[("track", "genre_top")].unique()
tracks_info.describe().T
percent_missing_data = tracks_info.isna().sum()/ len(tracks_info)* 100
percent_missing_data
columns_to_keep = percent_missing_data[percent_missing_data < 85].index
tracks_info[columns_to_keep]
track_id_to_genre = tracks_info[[("Unnamed: 0_level_0", "Unnamed: 0_level_1"), ("track", "genre_top")]]
tracks_info.head()
columns_to_remove = [("Unnamed: 0_level_0", "Unnamed: 0_level_1"), ("album", "id"), ("album", "information"), ("album", "tags"), ("album", "title"), ("album", "producer"), ("artist", "bio"), ("artist", "id"), ("artist", "latitude"), ("artist", "longitude"), ("artist", "location"), ("artist", "members"), ("artist", "name"), ("artist", "tags"), ("artist", "website"), ("track", "genres"), ("track", "genres_all"), ("track", "license"), ("track", "tags"), ("track", "title")]
columns_to_keep_manual_selection = set(tracks_info.columns).difference(set(columns_to_remove))
tracks_info = tracks_info[columns_to_keep_manual_selection]
sets = tracks_info[[("set", "subset"), ("set", "split")]]
tracks_info = tracks_info.drop([("set", "subset"), ("set", "split")], axis = 1)
tracks_info
tracks_info.columns
tracks_info
pd.to_datetime(tracks_info[("artist", "date_created")])
tracks_info.columns
tracks_info.dtypes
datetime_variables = [("track", "date_created"), ("artist", "date_created"), ("artist", "active_year_begin"), ("album", "date_created"), ("album", "date_released")]
datetime_variables_processed = pd.DataFrame()
for datetime_variable in datetime_variables:
    date = pd.to_datetime(tracks_info[datetime_variable])
    
    column_name = f"{datetime_variable[0]}_{datetime_variable[1]}"
    datetime_variables_processed[column_name + "_year"] = date.dt.year
    datetime_variables_processed[column_name + "_month"] = date.dt.month
tracks_info = pd.merge(left = tracks_info, right = datetime_variables_processed, left_on = tracks_info.index, right_on = datetime_variables_processed.index, )
tracks_info.columns
tracks_info = tracks_info.drop(datetime_variables, axis = 1)
tracks_info = tracks_info.drop(["key_0"], axis = 1)
tracks_info.dtypes
genres = tracks_info[("track", "genre_top")]
track_attributes = tracks_info.drop(("track", "genre_top"), axis = 1)
track_attributes = pd.get_dummies(track_attributes)
track_attributes = track_attributes.fillna(0)
track_attributes.describe().T
genres.unique()
genres.groupby(genres).size().sort_values(ascending = False)
len(genres[genres.isna()]) / len(genres)
genres = genres[~genres.isna()]
track_attributes = track_attributes.loc[genres.index]
len(genres), len(track_attributes)
track_attributes_scaled = MinMaxScaler().fit_transform(track_attributes)
track_attributes_train, track_attributes_test, genres_train, genres_test = train_test_split(
    track_attributes_scaled,
    genres,
    test_size = 0.15,
    stratify = genres)
track_attributes_train.shape, track_attributes_test.shape, genres_train.shape, genres_test.shape
logistic_regression = LogisticRegression(C = 1, max_iter = 1000)
logistic_regression.fit(track_attributes_train, genres_train)
def display_scores(estimator, train_attributes, train_labels, test_attributes, test_labels):
    print(f"Train score: {estimator.score(train_attributes, train_labels)}")
    print(f"Test score: {estimator.score(test_attributes, test_labels)}")    
display_scores(logistic_regression, track_attributes_train, genres_train, track_attributes_test, genres_test)
test_predictions = logistic_regression.predict(track_attributes_test)
cm = confusion_matrix(genres_test, test_predictions)
plt.figure(figsize = (10, 10))
sns.heatmap(
    cm, 
    annot = True, 
    fmt = "d", 
    cmap = plt.cm.Blues, 
    xticklabels = logistic_regression.classes_,
    yticklabels = logistic_regression.classes_)
track_audio_features = pd.read_csv("data/fma/features.csv", header = [0, 1, 2, 3])
track_audio_features
track_ids = track_audio_features[("feature", "statistics", "number", "track_id")]
tracks_for_modelling = track_id_to_genre[track_id_to_genre[("track", "genre_top")].isin(["Rock", "Hip-Hop", "Instrumental"])]
track_ids_for_modelling = tracks_for_modelling[("Unnamed: 0_level_0", "Unnamed: 0_level_1")]
track_genres_for_modelling = tracks_for_modelling[("track", "genre_top")]
track_ids_for_modelling
track_audio_features_for_modelling = track_audio_features[track_ids.isin(track_ids_for_modelling)]
track_audio_features_for_modelling = track_audio_features_for_modelling.drop([("feature", "statistics", "number", "track_id")], axis = 1)
track_audio_features_for_modelling = MinMaxScaler().fit_transform(track_audio_features_for_modelling)
track_audio_features_train, track_audio_features_test, genres_af_train, genres_af_test = train_test_split(
    track_audio_features_for_modelling,
    track_genres_for_modelling,
    test_size = 3500,
    stratify = track_genres_for_modelling
)
track_audio_features_train.shape, track_audio_features_test.shape, genres_af_train.shape, genres_af_test.shape
pca = PCA(n_components = 2)
pca.fit(track_audio_features_train)
pcs = pca.transform(track_audio_features_train)
first_pc, second_pc = pcs[:, 0], pcs[:, 1]
genre_colors = LabelEncoder().fit_transform(genres_af_train)
plt.scatter(first_pc, second_pc, c = genre_colors, s = 1)
plt.xlabel("First PC")
plt.ylabel("Second PC")
plt.show()
logistic_regression_af = LogisticRegression(max_iter = 1000)
logistic_regression_af.fit(track_audio_features_train, genres_af_train)
display_scores(logistic_regression_af, track_audio_features_train, genres_af_train, track_audio_features_test, genres_af_test)
def plot_confusion_matrix(estimator, attributes, labels, title = None):
    predictions = estimator.predict(attributes)
    cm = confusion_matrix(labels, predictions, normalize = "all")
    
    plt.figure(figsize = (5, 5))
    if title is not None:
        plt.title(title)
    sns.heatmap(
        cm, 
        annot = cm, 
        fmt = ".3%", 
        cmap = plt.cm.Blues, 
        xticklabels = estimator.classes_,
        yticklabels = estimator.classes_)
    plt.show()
plot_confusion_matrix(logistic_regression_af, track_audio_features_train, genres_af_train, title = "CM: train data")
plot_confusion_matrix(logistic_regression_af, track_audio_features_test, genres_af_test, title = "CM: test data")
print(classification_report(genres_af_test, logistic_regression_af.predict(track_audio_features_test)))
params_lr = {
    "solver": ["liblinear"],
    "penalty": ["l1"],
#     "C": 10 ** np.arange(-4, 5, dtype = np.float64)
    "C":[1e-2, 1e-3]
}
grid_search_lr = GridSearchCV(LogisticRegression(), params_lr, cv = 5)
grid_search_lr.fit(track_audio_features_train, genres_af_train)
grid_search_lr.best_estimator_
grid_search_lr.cv_results_
grid_search_lr.cv_results_["params"]
display_scores(grid_search_lr.best_estimator_, track_audio_features_train, genres_af_train, track_audio_features_test, genres_af_test)
len(grid_search_lr.best_estimator_.coef_[grid_search_lr.best_estimator_.coef_ != 0])
pca_af = PCA(n_components = 20)
pca_af.fit(track_audio_features_train)
track_audio_features_pca_train = pca_af.transform(track_audio_features_train)
track_audio_features_pca_test = pca_af.transform(track_audio_features_test)
plt.plot(pca_af.explained_variance_ratio_)
params_lr = {
    "solver": ["liblinear"],
    "penalty": ["l1"],
    "C": 10 ** np.arange(-4, 5, dtype = np.float64)
}
grid_search_lr = GridSearchCV(LogisticRegression(), params_lr, cv = 5)
grid_search_lr.fit(track_audio_features_pca_train, genres_af_train)
grid_search_lr.cv_results_
grid_search_lr.best_params_
grid_search_lr.best_estimator_.coef_
k_means = KMeans(n_clusters = 3)
k_means.fit(track_audio_features_pca_train)
plt.scatter(track_audio_features_pca_train[:, 0], track_audio_features_pca_train[:, 1], c = k_means.predict(track_audio_features_pca_train), s = 1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13214533672760934, 'fraction_numerical': 0.010414255959268688, 'mean_word_length': 8.059748427672956, 'pattern_counts': {'":': 7, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '352132', 'license': 'MIT', 'n_tokens_mistral': 3166, 'n_tokens_neox': 2992, 'n_words': 480, 'path': 'MachineLearning/08.ExamPrep/Genre Prediction Based on Music Features.ipynb', 'repo_name': 'LuGeorgiev/Python-SoftUni', 'size': '8642'}