<jupyter_start><jupyter_text>* * * *  Load Data<jupyter_code>df = pd.read_csv("../data/df_baseline.csv")
df = df.sort_values(by=['artist']).reset_index(drop=True)
print(df.shape)
df.head()
df.info()
df.describe()<jupyter_output><empty_output><jupyter_text>* * * *  Feature Preprocessing & Engineering  Rating  Scaling<jupyter_code>df['rating_pitch'] = df['rating_pitch'] * 10
df['rating_pitch'].head()
def avg_rating():
    for i in range(len(df)):
        if np.isnan(df.loc[i, 'rating_meta']) == True:
            if np.isnan(df.loc[i, 'rating_pitch']) == True:
                if np.isnan(df.loc[i, 'rating_AOTY']) == True:
                    df.loc[i, 'rating'] = 0
                else:
                    df.loc[i, 'rating'] = df.loc[i, 'rating_AOTY']
            else:
                df.loc[i, 'rating'] = df.loc[i, 'rating_pitch']
        else:
            df.loc[i, 'rating'] = df.loc[i, 'rating_meta']
            
    return df.head()
avg_rating()<jupyter_output><empty_output><jupyter_text>* * * *  Data Preparation for Modeling  Genre `hiphop`, `R&B`, `Soul`, `Funk`, `Pop`<jupyter_code>df = pd.get_dummies(df, columns=['genre'])
df.columns<jupyter_output><empty_output><jupyter_text>Split train & test data<jupyter_code>feature_names = ['single_count', 
                 'freq_billboard', 'freq_genius', 'freq_theSource', 'freq_xxl',
                 'twitter', 'instagram', 'facebook',
                 'spotify', 'soundcloud', 'youtube',
                 'genre_funk', 'genre_hiphop', 'genre_pop', 'genre_rnb', 'genre_soul',
                 'rating'
                ]
dfX = df[feature_names].copy()
dfy = df['label'].copy()
dfX.tail()
dfy.tail()
X_train, X_test, y_train, y_test = train_test_split(dfX, dfy, test_size=0.25, random_state=0)<jupyter_output><empty_output><jupyter_text>* * * *  Modeling : Random Forest<jupyter_code>model = RandomForestClassifier(criterion='entropy', n_estimators=10, max_depth=10,
                                min_samples_split=5, min_samples_leaf=5
                                ).fit(X_train, y_train)
print(confusion_matrix(y_train, model.predict(X_train)))
print(classification_report(y_train, model.predict(X_train)))
print(confusion_matrix(y_test, model.predict(X_test)))
print(classification_report(y_test, model.predict(X_test)))
from sklearn.metrics import roc_curve
from sklearn.metrics import auc

fpr, tpr, thresholds = roc_curve(y_test, model.predict_proba(X_test)[:, 1])
import matplotlib.pyplot as plt


def draw_roc_auc(model):
    y_prob = model.predict_proba(X_test)

    pos_probs = [y_prob[i][1] for i in range(len(y_prob))]

    fpr, tpr, threshold = roc_curve(y_test, pos_probs)
    auc = roc_auc_score(y_test, pos_probs)

    # Plot ROC
    fig, ax = plt.subplots(figsize=(12,12))
    ax.set_title('Receiver Operating Characteristic')
    plt.plot(fpr, tpr, 'b',label='Model - AUC = %0.3f'% auc)
    ax.legend(loc='lower right')
    plt.plot([0,1],[0,1],'r--', label='Chance')
    ax.legend()
    ax.set_xlim([-0.1,1.0])
    ax.set_ylim([-0.1,1.01])
    ax.set_ylabel('True Positive Rate')
    ax.set_xlabel('False Positive Rate')
    plt.show()
draw_roc_auc(model)
for name, importance in zip(X_train.columns, model.feature_importances_):
    print(name, importance)
scores_dict = {num_trees:[] for num_trees in np.logspace(start=1,stop=3.478,num=20).astype(int)}
oob_err_dict = {num_trees:0 for num_trees in np.logspace(start=1,stop=3.478,num=20).astype(int)}
k = 10
for num_trees in np.logspace(start=1,stop=3.478,num=20).astype(int):
    warnings.filterwarnings("ignore")
    clf = RandomForestClassifier(n_estimators=num_trees, n_jobs=-1, max_depth=10, oob_score=True)
    scores = cross_val_score(clf, X_train, y_train, cv=k, n_jobs=-1)
    model = clf.fit(X_train, y_train)
    preds = model.predict(X_test)
    precision = precision_score(y_test, preds)
    recall = recall_score(y_test, preds)
    f1 = f1_score(y_test, preds)
    scores_dict[num_trees].append([scores.mean(), precision, recall, f1])
    temp_oob = 1 - clf.oob_score_
    oob_err_dict[num_trees] = temp_oob
fig, ax = plt.subplots(figsize=(12,10))
ax.semilogx(list(scores_dict.keys()),[elem[0][0] for elem in scores_dict.values()], '-', label='Mean 10-Fold Accuracy')
ax.semilogx(list(scores_dict.keys()),[elem[0][1] for elem in scores_dict.values()], '-', label='Precision')
ax.semilogx(list(scores_dict.keys()),[elem[0][2] for elem in scores_dict.values()], '-', label='Recall')
ax.semilogx(list(scores_dict.keys()),[elem[0][3] for elem in scores_dict.values()], '-', label='F1')
ax.legend()
ax.set_xlabel('Number of Trees')
ax.set_ylabel('Measure')
# ax.set_ylim(0.89,1.)
plt.show()<jupyter_output><empty_output><jupyter_text>Recall (Sensitivity) $ \dfrac{TP}{TP+FN} $ 이 매우 중요다고 판단할 수 있다.**Precision의 경우**- False Positive :    - 무시해도 될 신인인데 굳이 컨텐츠를 생산했다.         - 뭐 어때, 에너지 소비하긴 했지만 괜찮.**Recall의 경우**- False Negative:    - 빠르게 컨텐츠를 생산해야 하는 신인인데, 걸러버렸다.        - 다른 매체에 선수를 뺐길 수 있다. 유저 유입이 적어짐.        - 다른 부가 한국에서의 공연 에이전시 계약, 머천다이즈 등 부가적인 기회들을 놓쳐버릴 수 있음. * * * *  Tackle Imbalance Problem<jupyter_code>from imblearn.under_sampling import *
for us in [ClusterCentroids, RandomUnderSampler, CondensedNearestNeighbour, EditedNearestNeighbours, RepeatedEditedNearestNeighbours, AllKNN, InstanceHardnessThreshold, NearMiss, NeighbourhoodCleaningRule, OneSidedSelection, TomekLinks]:
    print("{} Metrics : \n".format(us.__name__))
    rus = us(random_state=0)
    X_resampled_rus, y_resampled_rus = rus.fit_sample(X_train, y_train)
    model_tmp = RandomForestClassifier(criterion='entropy', n_estimators=10, max_depth=10,
                                min_samples_split=2 , min_samples_leaf=1
                                ).fit(X_resampled_rus, y_resampled_rus)
    print("Confusion Matrix \n".format(us.__name__), confusion_matrix(y_test, model_tmp.predict(X_test)), "\n")
    print("Classification Report\n".format(us.__name__), classification_report(y_test, model_tmp.predict(X_test)))
    print("AUC:".format(us.__name__), roc_auc_score(y_test, model_tmp.predict_proba(X_test)[:, 1]), "\n \n")
    "\n"
    "\n"<jupyter_output>ClusterCentroids Metrics : 

Confusion Matrix 
 [[151  64]
 [  3  53]] 

Classification Report
              precision    recall  f1-score   support

          0       0.98      0.70      0.82       215
          1       0.45      0.95      0.61        56

avg / total       0.87      0.75      0.78       271

AUC: 0.8604235880398671 
 

RandomUnderSampler Metrics : 

Confusion Matrix 
 [[191  24]
 [  7  49]] 

Classification Report
              precision    recall  f1-score   support

          0       0.96      0.89      0.92       215
          1       0.67      0.88      0.76        56

avg / total       0.90      0.89      0.89       271

AUC: 0.9429401993355482 
 

CondensedNearestNeighbour Metrics : 

Confusion Matrix 
 [[186  29]
 [ 11  45]] 

Classification Report
              precision    recall  f1-score   support

          0       0.94      0.87      0.90       215
          1       0.61      0.80      0.69        56

avg / total       0.87      0.85      0.86       271

[...]<jupyter_text>* * * *  Random Under Sampling<jupyter_code>from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(dfX, dfy, test_size=0.25, random_state=0)
rus = RandomUnderSampler(random_state=0)
X_resampled_rus, y_resampled_rus = rus.fit_sample(X_train, y_train)<jupyter_output><empty_output><jupyter_text>grid search on number of trees<jupyter_code>scores_dict = {num_trees:[] for num_trees in np.logspace(start=1,stop=3.478,num=20).astype(int)}
oob_err_dict = {num_trees:0 for num_trees in np.logspace(start=1,stop=3.478,num=20).astype(int)}
k = 10
for num_trees in np.logspace(start=1,stop=3.478,num=20).astype(int):
    warnings.filterwarnings("ignore")
    clf = RandomForestClassifier(n_estimators=num_trees, n_jobs=-1, max_depth=10, oob_score=True)
    scores = cross_val_score(clf, X_resampled_rus, y_resampled_rus, cv=k, n_jobs=-1)
    model = clf.fit(X_resampled_rus, y_resampled_rus)
    preds = model.predict(X_test)
    precision = precision_score(y_test, preds)
    recall = recall_score(y_test, preds)
    f1 = f1_score(y_test, preds)
    scores_dict[num_trees].append([scores.mean(), precision, recall, f1])
    temp_oob = 1 - clf.oob_score_
    oob_err_dict[num_trees] = temp_oob
fig, ax = plt.subplots(figsize=(12,10))
ax.semilogx(list(scores_dict.keys()),[elem[0][0] for elem in scores_dict.values()], '-', label='Mean 10-Fold Accuracy')
ax.semilogx(list(scores_dict.keys()),[elem[0][1] for elem in scores_dict.values()], '-', label='Precision')
ax.semilogx(list(scores_dict.keys()),[elem[0][2] for elem in scores_dict.values()], '-', label='Recall')
ax.semilogx(list(scores_dict.keys()),[elem[0][3] for elem in scores_dict.values()], '-', label='F1')
ax.legend()
ax.set_xlabel('Number of trees')
ax.set_ylabel('Measure')
plt.show()<jupyter_output><empty_output><jupyter_text>- 80% 이상라는 Recall의 기준은 달성했다.- 다음 단계는 Precision이 최고일 때의 파라미터를 찾는 것이다. 80~90개 사이의 tree에서 precision이 최고치를 나타내는 것을 볼 수 있다.  Grid Search<jupyter_code>from sklearn.model_selection import GridSearchCV
rfc = RandomForestClassifier()
param_grid = {
    'bootstrap': [True],
     'class_weight': [None],
    'criterion': ['entropy', 'gini'],
    'max_depth': [None, 3, 5, 7, 10, 12, 15],
    'max_features': ['auto'],
    'max_leaf_nodes': [None],
    'min_impurity_decrease': [0.0],
    'min_impurity_split': [None],
    'min_samples_leaf': [1, 2, 3, 4, 5, 7],
    'min_samples_split': [2, 3, 4, 5],
    'min_weight_fraction_leaf': [0.0],
    'n_estimators': [3, 5, 7, 10, 13, 15, 17, 20, 23, 25],
    'random_state': [None],
     'oob_score': [False],
     'verbose': [0],
     'warm_start': [False]
}
gs = GridSearchCV(estimator=rfc, param_grid=param_grid, scoring='recall', cv=10, n_jobs=1)
%%time
gs_result = gs.fit(X_resampled_rus, y_resampled_rus)
gs.best_params_<jupyter_output><empty_output><jupyter_text>Apply GridSearchCV Result and Tune Hyperparameter<jupyter_code>model = RandomForestClassifier(criterion='entropy', n_estimators=10, max_depth=10,
                                min_samples_split=2, min_samples_leaf=1
                                ).fit(X_resampled_rus, y_resampled_rus)
print(confusion_matrix(y_resampled_rus, model.predict(X_resampled_rus)))
print(classification_report(y_resampled_rus, model.predict(X_resampled_rus)))
print(confusion_matrix(y_test, model.predict(X_test)))
print(classification_report(y_test, model.predict(X_test)))
draw_roc_auc(model)<jupyter_output><empty_output><jupyter_text>Save Feature Importance for Visualization<jupyter_code>feature_importance = pd.DataFrame(columns=['feature', 'importance'])

for name, importance in zip(X_train.columns, model.feature_importances_):
    
    data = {
        'feature' : name,
        'importance' : importance
    }
    
    feature_importance.loc[len(feature_importance)] = data
feature_importance.sort_values(by=['importance'], ascending=False).reset_index(drop=True, inplace=True)
feature_importance
feature_importance.to_csv("df-feature_importance.csv", encoding='utf-8', index=False)<jupyter_output><empty_output><jupyter_text>Save Model<jupyter_code>import pickle

pickle.dump(model, open("rfc_rus_gs.plk", "wb"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14295843304085792, 'fraction_numerical': 0.033235841531507576, 'mean_word_length': 3.9167755991285405, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '614708', 'license': 'MIT', 'n_tokens_mistral': 4677, 'n_tokens_neox': 4492, 'n_words': 748, 'path': 'modeling/Final_Modeling.ipynb', 'repo_name': 'lucaseo/content-worth-debut-artist-classification-project', 'size': '11283'}