<jupyter_start><jupyter_text>Picking DF_Genres Dataframe<jupyter_code>pickle.dump(df_genres, open("./data/DF_Genres_List_v1.pkl","wb"))

# pickle.dump( df_genres_hashed, open( "DF_genres_hashed.pkl", "wb" ) )
df_genres["genres"].value_counts()<jupyter_output><empty_output><jupyter_text>Feature Engineering Goal - have only the highest occuring genre in 'genres' column.<jupyter_code>df_genres["genres"].value_counts()[0:4]
df["genres"][0:1]
((df_genres["genres"].value_counts().index=='rock').tolist()).index(False)
unique_genres = set(df_genres["genres"])
len(unique_genres)
df_genres
df.head()<jupyter_output><empty_output><jupyter_text>Feature Hashing<jupyter_code>from sklearn.feature_extraction import FeatureHasher
fh = FeatureHasher(input_type='string',n_features=50)
hashed_features = fh.fit_transform(df['genres_stripped'])
hashed_features = hashed_features.toarray()
df_genres_hashed = pd.concat([df, pd.DataFrame(hashed_features)], 
          axis=1)
pickle.dump( df_genres_hashed, open( "DF_genres_hashed.pkl", "wb" ) )<jupyter_output><empty_output><jupyter_text>Genre Clustering<jupyter_code>df["genres_stripped"]<jupyter_output><empty_output><jupyter_text>Embedding<jupyter_code>import ast
import IPython.display as ipd
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn.preprocessing import Normalizer, MinMaxScaler
from sklearn.cluster import MeanShift, KMeans, AffinityPropagation, AgglomerativeClustering, FeatureAgglomeration, SpectralClustering, estimate_bandwidth
from sklearn.pipeline import Pipeline
from sklearn.decomposition import PCA
from sklearn.metrics import adjusted_rand_score, adjusted_mutual_info_score, silhouette_score, v_measure_score
audio_df = df.drop(["artist","songid","genres","genres_stripped","track"],axis=1)
audio_columns = audio_df.columns
audio_columns
audio_features_columns = audio_columns[0:13]
audio_genres_columns = audio_columns[13:]
df[audio_features_columns]
# scale tempo
tempo = df['tempo'].values.reshape(-1,1)
tempo_scaled = MinMaxScaler().fit_transform(tempo)

# normalize audio columns
norm = Normalizer()
audio_features_normalized = norm.fit_transform(df[audio_features_columns])
norm.fit_transform(df[audio_features_columns])
data_for_model = np.concatenate((tempo_scaled,audio_features_normalized,df[audio_genres_columns]),axis=1)

X = data_for_model

Ks = np.arange(1, 20)
[KMeans(n_clusters=i, random_state=1986) for i in Ks]
km = [KMeans(n_clusters=i, random_state=1986) for i in Ks]
score = [-km[i].fit(X).score(X) for i in range(len(km))]
Ks.dtype = int
plt.figure(figsize=(8,8))
plt.plot(Ks,score)
plt.xlim(0,18)
plt.grid(True)
plt.xlabel('K')
plt.ylabel('Error')
plt.title('Elbow Method')
#K-means clustering doesn't appear to be a good choice here. 

# km = KMeans(n_clusters=9, n_init=10, random_state=1986)
# km.fit(X)
# labels = km.labels_
# cluster_centers = km.cluster_centers_
# labels_unique = np.unique(labels)
# n_clusters = len(labels_unique)
# km_clusters = pd.DataFrame(cluster_centers, columns=audio_cols)
# km_clusters['cluster'] = ['KM{}'.format(l) for l in labels_unique]
# km_clusters['method'] = 'KMeans'
# clusters = km_clusters
# cols = ['cluster'] + audio_cols + ['method']
# clusters = clusters[cols]
# clusters
# plt.figure(figsize=(10,10))
# sns.set(font_scale=1.5)
# sns.heatmap(
#     data=clusters.loc[clusters.method=='KMeans', audio_cols],
#     cmap='Purples',
#     annot=True
# )
# plt.ylabel("KMeansLabel")
# plt.savefig('heatmap_all.png')
pickle.dump( X, open( "./data/X_array.pkl", "wb" ) )<jupyter_output><empty_output><jupyter_text>DBSCAN Attempt - Takes too long on local machine - Will try Sagemaker<jupyter_code>from sklearn.cluster import DBSCAN

dbscan = DBSCAN(eps=3, metric='euclidean', min_samples=50, n_jobs=-1) 
dbscan.fit(X)
len(dbscan.labels_)
len(set(dbscan.labels_))
dbscan.labels_[1]
np.where(dbscan.labels_ == 1)[0]
for result in np.where(dbscan.labels_ == 1)[0]:
    display(pd.DataFrame(df.loc[result][10:11]),pd.DataFrame(df.loc[result][17:18]))<jupyter_output><empty_output><jupyter_text>Genre Replace with 200 > Value_Counts<jupyter_code>unique_genres = df_genres["genres"].value_counts().index
unique_genres
df_genres["genres"].value_counts()
high_occuring = []
for i in df_genres["genres"].value_counts().iteritems():
    if high_occuring == []:
        high_occuring.append(i)
        print("list_appended!")
        print(high_occuring)
        continue
    if i[1] > high_occuring[0][1]:
        vals = [(val)]
        print(vals)
    else:
        continue

print(high_occuring[0][0])
vals
df_genres["genres"].value_counts()[0:10]
vals[0][1]
df_genres["genres"][df_genres["genres"].value_counts() > 200]
high_occuring = df_genres["genres"].value_counts()[df_genres["genres"].value_counts() > 100]
high_occuring_list = high_occuring.index
def convert_to_list(string):
    return list(string.split(","))
df["genres_stripped"][0]
df["genres_stripped"][0].split(",")
if "suomi rock" in df["pop_genres"][0]:
    print("yes")
df["pop_genres"]
df["genres_stripped"][0]
df["pop_genres"] = df["genres_stripped"].apply(convert_to_list)
value = 'suomi rock'
value = "'" + value + "'"
df_genres["genres"].value_counts()[df_genres["genres"].value_counts().index == value ]
df_genres["genres"].value_counts().index == "'rock'"
for item in df["pop_genres"][0:5]:
    high_occuring = []
    for genre in item:
        genre = genre.strip(" ")
        for i in df_genres["genres"].value_counts()[df_genres["genres"].value_counts().index == genre].iteritems():
            print(i)
            if high_occuring == []:
                high_occuring = [(i)]
                print("added 1st time")
                continue
            if i[1] > high_occuring[0][1]:
                print("replaced!")
                high_occuring = [(i)] 
            else:
                continue

    print(high_occuring[0][0])
    
def keep_highest_occuring(list):
    high_occuring = []
    for genre in list:
        genre = genre.strip(" ")   
        for i in df_genres["genres"].value_counts()[df_genres["genres"].value_counts().index == genre].iteritems():
            if high_occuring == []:
                high_occuring = [(i)]
                continue
            if i[1] > high_occuring[0][1]:
                high_occuring = [(i)] 
            else:
                continue
    print (high_occuring[0][0])    
    return high_occuring[0][0]
df["most_pop_genre"] = df["pop_genres"][:10].apply(keep_highest_occuring)
def keep_highest_occuring_2(list):
    val_counts_array = np.array(df_genres["genres"].value_counts().reset_index())
    high_occuring = []
    for genre in list:
        genre = genre.strip(" ")   
        for i in val_counts_array:
            if i[0] == genre:
                if high_occuring == []:
                    high_occuring = [(i)]
                    continue
                if i[1] > high_occuring[0][1]:
                    high_occuring = [(i)] 
                else:
                    continue
    return high_occuring[0][0]
df["most_pop_genre"] = df["pop_genres"].apply(keep_highest_occuring_2)
df["most_pop_genre"][0:100]
def keep_highest_occuring(list):
    high_occuring = []
    for i in df_genres["genres"].value_counts().iteritems():
        if high_occuring == []:
            high_occuring.append(i)
            print("list_appended!")
            print(high_occuring)
            continue
        if i[1] > high_occuring[0][1]:
            vals = [(val)] 
        else:
            continue

    return(high_occuring[0][0])
df[["genres","pop_genres"]][df["genres"].str.contains("finnish indie")]
for item in df["genres"][0:4]:
    print(item)
#     high_occuring = []
#     for song in item:
#         for 
        
high_occuring_list
df["pop_genres"][0:1].apply(remove_lower_occuring) 
for item in df["genres_stripped"][0:20]:
    print(item)
for item in df["pop_genres"][0:100]:
    print(item)
df["pop_genres"].isna().sum()
df.columns
with open('./DF_genres_hashed.pkl', 'rb') as f:
    df = pickle.load(f)
genre<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14852467146045129, 'fraction_numerical': 0.01624101165385569, 'mean_word_length': 4.3815877251501, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '635293', 'license': 'MIT', 'n_tokens_mistral': 3062, 'n_tokens_neox': 2903, 'n_words': 467, 'path': 'model_notebooks/_archive/_archive_genre_encoding.ipynb', 'repo_name': 'BloomTech-Labs/sound-drip-ds-x', 'size': '8066'}