<jupyter_start><jupyter_text>Read data<jupyter_code>train_old = pd.read_csv("../data/train_mfcc.csv")
train_old.head() 
df = pd.read_csv("../data/train_new_feat.csv") 
df['genre'] = train_old[' genre']
df.head()<jupyter_output><empty_output><jupyter_text>Data cleaning<jupyter_code>#Drop useless features
useless = [ ' mother tongue', ' liked', ' disliked', ' amazement',' mood',' age', ' gender',' solemnity', ' nostalgia']
df  = df.drop(useless, 1)
df = pd.get_dummies(df, columns=['genre'])
df.head()
LABELS =  [' tenderness', ' calmness', ' power',
       ' joyful_activation', ' tension', ' sadness',
           "genre_classical", "genre_electronic", "genre_pop", "genre_rock"
         ]
#Get list of base Features
FEATURES = df.columns.drop(['track id'] +  LABELS)
print(f"base features: {FEATURES.tolist()}")
# drop duplicates to avoid ovefitting
Xtrain = df[FEATURES.tolist() + ['track id']].drop_duplicates()<jupyter_output><empty_output><jupyter_text>Feature engineering As previously observed when performing the EDA,  all the subjects do not share the same feelings for the same song and most of the subjects did not feel any emotion.Thus I decide to proceed as follows: For a given song, if more than 30% of the subjects indicate felt the same emotion, then<jupyter_code>ytrain = df.groupby('track id')[LABELS].mean() 
ytrain = ytrain.reset_index()

for l in LABELS:
    ytrain[l] = ytrain[l].apply(lambda x: 1 if x > 0.3 else 0, 1)
train = Xtrain.merge(ytrain, on='track id', how='left')<jupyter_output><empty_output><jupyter_text>Outlier treatment<jupyter_code>from pyod.models.iforest import IForest 
from pyod.models.knn import KNN 

# predict and treat outliers for a given feature 
def predict_and_treat_outlier(feat,db):
    """
    This function treats outliers for a given feature 
    
    Parameters:
        db (DataFrame): the dataframe containing the outliers to be cleaned
        feat (string): the given feature 
    Returns:
        db (DataFrame): the  dataframe with cleaned outliers
    """
  X =  db[[feat]]
  clf = IForest() #KNN()
  clf.fit(X)
  db['is_outlier'] = clf.predict(X)
  db_cleaned = db.copy()
  mask = db_cleaned['is_outlier'] == 1
  db_cleaned.loc[mask, feat] = db_cleaned[feat].median()
  return db_cleaned.drop(columns=['is_outlier'])
# predict and treat age outliers 
train = predict_and_treat_outlier(' age',train) 
train.head()
# predict audio features outliers  
def predict_audio_feat_outlier(db):
    """
    This function globally detects outliers  
    
    Parameters:
        db (DataFrame): the dataframe containing the values to be cleaned
    Returns:
        db_tr (DataFrame): the  dataframe with detected outliers
    """
    features = ['chroma_sftf', 'rolloff', 'zero_crossing_rate', 'rmse', 'flux', 'contrast', 'flatness',
                          'sample_silence', 'mfcc_0', 'mfcc_1','mfcc_2', 'mfcc_3', 'mfcc_4', 'mfcc_5', 'mfcc_6',
                          'mfcc_7', 'mfcc_8', 'mfcc_9', 'mfcc_10','mfcc_11','mfcc_12', 'mfcc_13', 'mfcc_14',
                          'mfcc_15', 'mfcc_16', 'mfcc_17', 'mfcc_18', 'mfcc_19', 'tempo']
    db_tr = db.copy() 
    clf =  IForest() 
    clf.fit(db_tr[features])
    db_tr['is_outlier'] = clf.predict(db_tr[features])
    
    return db_tr
#treat audio features outliers
def treat_audio_feat_outlier(db):
    """
    This function globally treats outliers  
    
    Parameters:
        db (DataFrame): the dataframe containing the outliers to be cleaned
    Returns:
        new_db (DataFrame): the  dataframe with cleaned outliers
    """
    features = ['chroma_sftf', 'rolloff', 'zero_crossing_rate', 'rmse', 'flux', 'contrast', 'flatness',
                          'sample_silence', 'mfcc_0', 'mfcc_1','mfcc_2', 'mfcc_3', 'mfcc_4', 'mfcc_5', 'mfcc_6',
                          'mfcc_7', 'mfcc_8', 'mfcc_9', 'mfcc_10','mfcc_11','mfcc_12', 'mfcc_13', 'mfcc_14',
                          'mfcc_15', 'mfcc_16', 'mfcc_17', 'mfcc_18', 'mfcc_19', 'tempo']
    mask = db['is_outlier'] == 1
    
    for f in features: 
        db.loc[mask, f] = db[f].median()
    return db.drop(columns=['is_outlier'])
# predict audio features outliers
train = predict_audio_feat_outlier(train)
# treat audio features outliers
train = treat_audio_feat_outlier(train)
train.head()<jupyter_output><empty_output><jupyter_text>Model training : MLP model<jupyter_code># Data normalization
scaler = MinMaxScaler()
train[FEATURES] = scaler.fit_transform(train[FEATURES])<jupyter_output><empty_output><jupyter_text>Model Validation<jupyter_code># Model Validation 
epochs = 5
kf = KFold(n_splits=epochs, shuffle=True, random_state=1997)  # 30, n_split 3
X_train = train[FEATURES] 
y_train = train[LABELS]
y_oof = np.zeros([X_train.shape[0], len(LABELS)])
i = 0
for tr_idx, val_idx in kf.split(X_train, y_train):
    X_tr, X_vl = X_train.iloc[tr_idx, :], X_train.iloc[val_idx, :]
    y_tr, y_vl = y_train.iloc[tr_idx, :], y_train.iloc[val_idx, :]
    model = MLPClassifier([300], random_state=47, activation='logistic', max_iter=5000)
    model.fit(X_tr, y_tr) 
    y_pred = model.predict_proba(X_vl)
    y_oof[val_idx, :] = y_pred
    i += 1
    acc = roc_auc_score(y_vl, y_pred, multi_class='ovr')
    print(f"Fold #{i} AUC : {round(acc, 2)}")
metric = roc_auc_score(y_train, y_oof, multi_class='ovr')
print(f"Full AUC  : {round(metric, 2)}")<jupyter_output>Fold #1 AUC : 0.79
Fold #2 AUC : 0.75
Fold #3 AUC : 0.76
Fold #4 AUC : 0.78
Fold #5 AUC : 0.75
Full AUC  : 0.76<jupyter_text>Save model and scaler<jupyter_code>import pickle
filename = '../model_saved/emo_genre_clf_model.sav'
pickle.dump(model, open(filename, 'wb'))

filename = '../model_saved/Scaler_Extractor.sav'
pickle.dump(scaler, open(filename, 'wb'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14094076655052265, 'fraction_numerical': 0.02003484320557491, 'mean_word_length': 4.181407942238267, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '510939', 'license': 'MIT', 'n_tokens_mistral': 2235, 'n_tokens_neox': 2088, 'n_words': 475, 'path': 'notebook/Music_Genre_Emotion_Classification_Baseline_DNN .ipynb', 'repo_name': 'FanambyH/Final_Rec_Sys_App', 'size': '5740'}