<jupyter_start><jupyter_text>SONG POPULARITY DATA PREPARATION NOTEBOOK  LOADING THE NECESSARY LIBRARIES<jupyter_code># !pip install catboost
#Import some libraries
import pandas as pd

pd.options.display.max_rows = 2200
pd.options.display.max_columns = 2200


import numpy as np
import gc
import matplotlib.pyplot as plt
import plotly.express as px
from pylab import rcParams
import seaborn as sb
sb.set_style('darkgrid')
rcParams['figure.figsize'] = 8,8
import warnings
warnings.filterwarnings("ignore")
%matplotlib inline
from imblearn.over_sampling import SMOTE,RandomOverSampler
import lightgbm as lgb
import xgboost as xgb
from catboost import CatBoostClassifier as cat
from collections import Counter
from sklearn.metrics import plot_confusion_matrix,plot_roc_curve,roc_auc_score,log_loss,precision_score
from sklearn.preprocessing import LabelEncoder,PolynomialFeatures,StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.experimental import enable_iterative_imputer
from sklearn.impute import IterativeImputer,KNNImputer ,SimpleImputer
# #Start by connecting gdrive into the google colab
# from google.colab import drive
# drive.mount('/content/gdrive')
# path = "/content/gdrive/MyDrive/churn prediction"<jupyter_output><empty_output><jupyter_text>LOAD THE DATASETS<jupyter_code>#import data
# path+'/Train (10).csv'
train = pd.read_csv('train.csv')
test=  pd.read_csv('test.csv')
submission = pd.read_csv('sample_submission.csv')
train.head(2)
train.shape
#view the data types in the data data
train.info()
duplicate = train[train.duplicated()]
duplicate.shape<jupyter_output><empty_output><jupyter_text>EXPLORATORY DATA ANALYSIS<jupyter_code># check unique values in dataset
train.apply(lambda x: len(x.unique()))
#Check how many levels are involved in each of the categorical features (object)
print(train['key'].value_counts())
plt.figure(figsize=(10,5))
train['key'].value_counts(normalize=True).plot(kind='bar')
plt.ylabel('counts')
plt.xlabel('key')
print(train['audio_mode'].value_counts())
plt.figure(figsize=(10,5))
train['audio_mode'].value_counts(normalize=True).plot(kind='bar')
plt.ylabel('counts')
plt.xlabel('audio_mode')
print(train['time_signature'].value_counts())
plt.figure(figsize=(10,5))
train['time_signature'].value_counts(normalize=True).plot(kind='bar')
plt.ylabel('counts')
plt.xlabel('time_signature')
#Check if the predictor class is balanced 
print(train['song_popularity'].value_counts())
plt.figure(figsize=(10,5))
train['song_popularity'].value_counts(normalize=True).plot(kind='bar')
plt.ylabel('counts')
plt.xlabel('song_popularity')
#NUMERICAL DATA THAT IS DISCRETE IN NATURE
plt.figure(figsize = (13,5))
plt.subplot(1,2,1)
sb.countplot(x='key',hue='song_popularity',data=train,palette='husl')
plt.title("key VS song_popularity")


plt.figure(figsize = (13,5))
plt.subplot(1,2,1)
sb.countplot(x='audio_mode',hue='song_popularity',data=train,palette='husl')
plt.title("audio_mode VS song_popularity")

plt.figure(figsize = (13,5))
plt.subplot(1,2,1)
sb.countplot(x='time_signature',hue='song_popularity',data=train)
plt.title("time_signature VS song_popularity")

corr = train.corr()
sb.heatmap(corr)
#Check summary of numerical fields
train.select_dtypes(include=['int64', 'float64']).describe().T
# #id is not an important feature so we drop it
# train.drop(['id'], 1, inplace = True)
# test.drop([ 'id'], 1, inplace = True)<jupyter_output><empty_output><jupyter_text>DATA MANIPULATION<jupyter_code>ntrain = train.shape[0]
ntest = test.shape[0]
data = pd.concat((train, test)).reset_index(drop=True)
#check for missing values
percent_missing = data.isnull().sum() * 100 / len(data)
missing_value_data = pd.DataFrame({'column_name': data.columns,
                                 'percent_missing': percent_missing})
missing_value_data<jupyter_output><empty_output><jupyter_text>HANDLING MISSING DATA In real world data if you use sklearn impute methods you have to fit and transform the train data first then transform the test data separately because the test is unseen data, but in a compettion environment you can cheat by fit_transforming the whole data concatenated since you already are provided with the test data.<jupyter_code>#Key was a discrete numerical column so i filled it with 0.0 since it was the  most frequent
data['key'] = data['key'].fillna(0.0)<jupyter_output><empty_output><jupyter_text>ITERATIVE IMPUTER<jupyter_code>#i used iterative imputer for the continuous numerical columns
X1 = data[['song_duration_ms','liveness',  'danceability',  'instrumentalness','energy' ,'acousticness' ,'loudness']] 
# X2 = data[[ 'key']]

iter_im  = IterativeImputer(random_state =47 ,max_iter = 60)
X1_out = iter_im.fit_transform(X1)


data[['song_duration_ms','liveness',  'danceability',  'instrumentalness','energy' ,'acousticness' ,'loudness']] = X1_out
# data[[ 'key']] = X2_out
data.head()<jupyter_output><empty_output><jupyter_text>KNN IMPUTER<jupyter_code>#Wanted to use KNN imputer for categorical missing features but it takes a lot of time in my local machine which has a slow processor


# knn_im = KNNImputer(n_neighbors=5, weights='uniform')
# X2_out = knn_im.fit_transform(X2)
data.columns<jupyter_output><empty_output><jupyter_text>FEATURE ENGINEERING MEAN ENCODING<jupyter_code>KEY = data.groupby('key').mean()
KEY.drop(['song_popularity'], 1, inplace = True)
cols = []
for i in KEY.columns:
    if i != 'key':
        KEY[i+'_key_mean_all'] = KEY[i]
        KEY.drop(i, 1, inplace = True)
        cols.append(i+'_key_mean_all')

data = pd.merge(data, KEY, on='key', how = 'left')


AUDIOMODE = data.groupby('audio_mode').mean()
AUDIOMODE.drop(['song_popularity','acousticness_key_mean_all', 'audio_valence_key_mean_all',
       'danceability_key_mean_all', 'energy_key_mean_all',
       'instrumentalness_key_mean_all', 'liveness_key_mean_all',
       'loudness_key_mean_all', 'song_duration_ms_key_mean_all',
       'speechiness_key_mean_all', 'tempo_key_mean_all','audio_mode_key_mean_all','time_signature_key_mean_all'], 1, inplace = True)
cols = []
for i in AUDIOMODE.columns:
    if i != 'audio_mode':
        AUDIOMODE[i+'_audio_mode_mean_all'] = AUDIOMODE[i]
        AUDIOMODE.drop(i, 1, inplace = True)
        cols.append(i+'_audio_mode_mean_all')

data = pd.merge(data, AUDIOMODE, on='audio_mode', how = 'left')
TIME_SIGNATURE = data.groupby('time_signature').mean()
TIME_SIGNATURE.drop(['song_popularity','acousticness_key_mean_all', 'audio_valence_key_mean_all',
       'danceability_key_mean_all', 'energy_key_mean_all',
       'instrumentalness_key_mean_all', 'liveness_key_mean_all',
       'loudness_key_mean_all', 'song_duration_ms_key_mean_all',
       'speechiness_key_mean_all', 'tempo_key_mean_all','audio_mode_key_mean_all','time_signature_key_mean_all',
       'acousticness_audio_mode_mean_all', 'audio_valence_audio_mode_mean_all',
       'danceability_audio_mode_mean_all', 'energy_audio_mode_mean_all',
       'instrumentalness_audio_mode_mean_all', 'liveness_audio_mode_mean_all',
       'loudness_audio_mode_mean_all', 'song_duration_ms_audio_mode_mean_all',
       'speechiness_audio_mode_mean_all', 'tempo_audio_mode_mean_all','key_audio_mode_mean_all','time_signature_audio_mode_mean_all'], 1, inplace = True)
cols = []
for i in TIME_SIGNATURE.columns:
    if i != 'time_signature':
        TIME_SIGNATURE[i+'_time_signature_mean_all'] = TIME_SIGNATURE[i]
        TIME_SIGNATURE.drop(i, 1, inplace = True)
        cols.append(i+'_time_signature_mean_all')

data = pd.merge(data, TIME_SIGNATURE, on='time_signature', how = 'left')<jupyter_output><empty_output><jupyter_text>FEATAURE INTERACTIONS<jupyter_code>data['acousticness_key_mean_all_diff'] = data['acousticness'] - data['acousticness_key_mean_all']
data['acousticness_audio_mode_mean_all_diff'] = data['acousticness'] - data['acousticness_audio_mode_mean_all']
data['acousticness_time_signature_mean_all_diff'] = data['acousticness'] - data['acousticness_time_signature_mean_all']

data['audio_valence_key_mean_all_diff'] = data['audio_valence'] - data['audio_valence_key_mean_all']
data['audio_valence_audio_mode_mean_all_diff'] = data['audio_valence'] - data['audio_valence_audio_mode_mean_all']
data['audio_valence_time_signature_mean_all_diff'] = data['audio_valence'] - data['audio_valence_time_signature_mean_all']

data['danceability_key_mean_all_diff'] = data['danceability'] - data['danceability_key_mean_all']
data['danceability_audio_mode_mean_all_diff'] = data['danceability'] - data['danceability_audio_mode_mean_all']
data['danceability_time_signature_mean_all_diff'] = data['danceability'] - data['danceability_time_signature_mean_all']

data['energy_key_mean_all_diff'] = data['energy'] - data['energy_key_mean_all']
data['energy_audio_mode_mean_all_diff'] = data['energy'] - data['energy_audio_mode_mean_all']
data['energy_time_signature_mean_all_diff'] = data['energy'] - data['energy_time_signature_mean_all']

data['instrumentalness_key_mean_all_diff'] = data['instrumentalness'] - data['instrumentalness_key_mean_all']
data['instrumentalness_audio_mode_mean_all_diff'] = data['instrumentalness'] - data['instrumentalness_audio_mode_mean_all']
data['instrumentalness_time_signature_mean_all_diff'] = data['instrumentalness'] - data['instrumentalness_time_signature_mean_all']

data['liveness_key_mean_all_diff'] = data['liveness'] - data['liveness_key_mean_all']
data['liveness_audio_mode_mean_all_diff'] = data['liveness'] - data['liveness_audio_mode_mean_all']
data['liveness_time_signature_mean_all_diff'] = data['liveness'] - data['liveness_time_signature_mean_all']

data['loudness_key_mean_all_diff'] = data['loudness'] - data['loudness_key_mean_all']
data['loudness_audio_mode_mean_all_diff'] = data['loudness'] - data['loudness_audio_mode_mean_all']
data['loudness_time_signature_mean_all_diff'] = data['loudness'] - data['loudness_time_signature_mean_all']

data['song_duration_ms_key_mean_all_diff'] = data['song_duration_ms'] - data['song_duration_ms_key_mean_all']
data['song_duration_ms_audio_mode_mean_all_diff'] = data['song_duration_ms'] - data['song_duration_ms_audio_mode_mean_all']
data['song_duration_ms_time_signature_mean_all_diff'] = data['song_duration_ms'] - data['song_duration_ms_time_signature_mean_all']

data['speechiness_key_mean_all_diff'] = data['speechiness'] - data['speechiness_key_mean_all']
data['speechiness_audio_mode_mean_all_diff'] = data['speechiness'] - data['speechiness_audio_mode_mean_all']
data['speechiness_time_signature_mean_all_diff'] = data['speechiness'] - data['speechiness_time_signature_mean_all']

data['tempo_key_mean_all_diff'] = data['tempo'] - data['tempo_key_mean_all']
data['tempo_audio_mode_mean_all_diff'] = data['tempo'] - data['tempo_audio_mode_mean_all']
data['tempo_time_signature_mean_all_diff'] = data['tempo'] - data['tempo_time_signature_mean_all']


data['energy_loudness_sum'] = data['energy'] + data['loudness']
data['energy_loudness_diff'] = data['energy'] - data['loudness']

data['audvalence_danceability_diff'] = data['audio_valence'] - data['danceability']
data['audvalence_danceability_sum'] = data['audio_valence'] + data['danceability']

data['audvalence_loudness_sum'] = data['audio_valence'] + data['loudness']
data['audvalence_loudness_diff'] = data['audio_valence'] - data['loudness']

data['audvalence_loudness_diff'] = data['audio_valence'] - data['energy']
data['audvalence_loudness_sum'] = data['audio_valence'] + data['energy']

data['danceability_valence'] = data['danceability']+data['audio_valence']
data['danceability_valence_diff'] = data['danceability'] - data['audio_valence']


data['tempo_danceability'] = data['tempo'] + data['danceability']
data['tempo_danceability_diff'] = data['tempo'] - data['danceability']

data['speechiness_valence'] = data['speechiness'] + data['audio_valence']
data['speechiness_valence_diff'] = data['speechiness'] - data['audio_valence']

data['loudness_speechiness_diff'] = data['loudness'] - data['speechiness']
data['loudness_speechiness'] = data['loudness'] + data['speechiness']

data['audio_valence_speechiness'] = data['audio_valence'] + data['speechiness']
data['audio_valence_speechiness_diff'] = data['audio_valence'] - data['speechiness']



data['energy_acousticness_diff'] = data['energy'] - data['acousticness']
data['energy_acousticness'] = data['energy'] + data['acousticness']

data['loudness_acousticness_diff'] = data['loudness'] - data['acousticness']
data['loudness_acousticness'] = data['loudness'] + data['acousticness']

data['energy_danceability'] = data['energy']+ data['danceability']
data['energy_danceability_diff'] = data['energy'] - data['danceability']

data['instrumentalness_acousticness_diff'] = data['instrumentalness'] - data['acousticness']
data['instrumentalness_acousticness'] = data['instrumentalness'] + data['acousticness']



data['time_danceability'] = data['time_signature'] * data['danceability']
data['time_danceability_div'] = data['danceability'] / data['time_signature']

data['time_energy'] = data['time_signature'] * data['energy']
data['time_energy_div'] = data['energy'] / data['time_signature']

data['time_loudness'] = data['time_signature']* data['loudness']
data['time_loudness_div'] = data['loudness'] / data['time_signature'] 

data['key_instrumentalness '] = data['key'] * data['instrumentalness']


data['time_tempo'] = data['time_signature'] * data['tempo']
data['time_tempo_div'] = data['tempo'] / data['time_signature']

data['key_acousticness '] = data['key'] * data['acousticness']

data['superior_features'] = data['audio_valence'] + data['danceability'] + data['energy']
data['inferior_features'] = data['liveness'] + data['speechiness'] + data['acousticness']

data['time_signature_accousticness'] = data['time_signature'] + data['acousticness']
data['time_signature_accousticness_div'] =  data['acousticness']/data['time_signature']

data['danceability_instrumentalness'] = data['danceability'] + data['instrumentalness']
data['danceability_instrumentalness_diff'] = data['danceability'] - data['instrumentalness']

data['energy_audio_valence'] = data['energy'] + data['audio_valence']
data['energy_audio_valence_diff'] = data['energy'] - data['audio_valence']

data['energy_time_signature'] = data['energy'] + data['time_signature']

data['energy_liveness'] = data['energy'] + data['liveness']
data['energy_liveness_diff'] = data['energy'] - data['liveness']

data['energy_instrumentalness'] = data['energy'] + data['instrumentalness']
data['energy_instrumentalness_diff'] = data['energy'] - data['instrumentalness']
 
data['instrumentalness_audio_valence'] = data['instrumentalness'] + data['audio_valence']
data['instrumentalness_audio_valence_diff'] = data['instrumentalness'] - data['audio_valence']<jupyter_output><empty_output><jupyter_text>GETTING TIME FEATURES FROM SONG_DURATION_MS FEATURE<jupyter_code>def add_standard_time(data): 
    ms = data['song_duration_ms']
    millis = ms
    seconds=(millis/1000)%60
    minutes=(millis/(1000*60))%60
    hours=(millis/(1000*60*60))%24
    data['song_duration_s'] = seconds
    data['song_duration_m'] = minutes
    data['song_duration_h'] = hours
    return data

data = add_standard_time(data)

data.head()<jupyter_output><empty_output><jupyter_text>BINNING<jupyter_code># data['accousticness_bins'] = data['acousticness'].apply(lambda x:1 if x > 0.5 else 0)
# data['danceability_bins'] = data['danceability'].apply(lambda x:1 if x > 0.5 else 0)
# data['energy_bins'] = data['energy'].apply(lambda x:1 if x>0.5 else 0)
# data['instrumentalness_bins'] = data['instrumentalness'].apply(lambda x:1 if x>0.09 else 0)
# data['liveness_bins'] = data['liveness'].apply(lambda x:1 if x>0.2 else 0)
# data['loudness_bins']= data['loudness'].apply(lambda x:1 if x > -13 else 0)
# data['speechiness_bins']= data['speechiness'].apply(lambda x:1 if x>0.25 else 0)<jupyter_output><empty_output><jupyter_text>POLYNOMIAL FEATURES<jupyter_code># poly_cols = ['acousticness','audio_valence', 'danceability', 'energy','instrumentalness','liveness', 'loudness',
#        'song_duration_ms','speechiness', 'tempo']
# poly = PolynomialFeatures(degree=3, interaction_only=True, include_bias=False)
# data_poly = poly.fit_transform(data[poly_cols])
# df_poly = pd.DataFrame(data_poly, columns=[f"poly_{i}" for i in range(data_poly.shape[1])])
# data = pd.concat([data, df_poly], axis=1)
data.columns<jupyter_output><empty_output><jupyter_text>HANDLING OUTLIERS<jupyter_code>#finding outliers and skewness
plt.figure(figsize=(13,7))
plt.subplot(2,1,2)
sb.boxplot(data['acousticness'])
plt.title("boxplot of acousticness")
plt.show()
plt.subplot(2,1,2)
sb.distplot(data["acousticness"])
plt.title("distplot of acousticness")
plt.show()
plt.figure(figsize=(13,7))
plt.subplot(2,1,2)
sb.boxplot(data['audio_valence'])
plt.title("boxplot of audio_valence")
plt.show()
plt.subplot(2,1,2)
sb.distplot(data["audio_valence"])
plt.title("distplot of audio_valence")
plt.show()
plt.figure(figsize=(13,7))
plt.subplot(2,1,2)
sb.boxplot(data['danceability'])
plt.title("boxplot of danceability")
plt.show()
plt.subplot(2,1,2)
sb.distplot(data["danceability"])
plt.title("distplot of danceability")
plt.show()
plt.figure(figsize=(13,7))
plt.subplot(2,1,2)
sb.boxplot(data['energy'])
plt.title("boxplot of energy")
plt.show()
plt.subplot(2,1,2)
sb.distplot(data["energy"])
plt.title("distplot of energy")
plt.show()
plt.figure(figsize=(13,7))
plt.subplot(2,1,2)
sb.boxplot(data['instrumentalness'])
plt.title("boxplot of instrumentalness")
plt.show()
plt.subplot(2,1,2)
sb.distplot(data["instrumentalness"])
plt.title("distplot of instrumentalness")
plt.show()

plt.figure(figsize=(13,7))
plt.subplot(2,1,2)
sb.boxplot(data['liveness'])
plt.title("boxplot of liveness")
plt.show()
plt.subplot(2,1,2)
sb.distplot(data["liveness"])
plt.title("distplot of liveness")
plt.show()
plt.figure(figsize=(13,7))
plt.subplot(2,1,2)
sb.boxplot(data['loudness'])
plt.title("boxplot of loudness")
plt.show()
plt.subplot(2,1,2)
sb.distplot(data["loudness"])
plt.title("distplot of loudness")
plt.show()
plt.figure(figsize=(13,7))
plt.subplot(2,1,2)
sb.boxplot(data['song_duration_ms'])
plt.title("boxplot of song_duration_ms")
plt.show()
plt.subplot(2,1,2)
sb.distplot(data["song_duration_ms"])
plt.title("distplot of song_duration_ms")
plt.show()
plt.figure(figsize=(13,7))
plt.subplot(2,1,2)
sb.boxplot(data['speechiness'])
plt.title("boxplot of speechiness")
plt.show()
plt.subplot(2,1,2)
sb.distplot(data["speechiness"])
plt.title("distplot of speechiness")
plt.show()
plt.figure(figsize=(13,7))
plt.subplot(2,1,2)
sb.boxplot(data['tempo'])
plt.title("boxplot of tempo")
plt.show()
plt.subplot(2,1,2)
sb.distplot(data["tempo"])
plt.title("distplot of tempo")
plt.show()
data[['liveness','acousticness','loudness','speechiness','tempo']].describe()
#Instead of dropping the outliers i decided to cap them with the 0.05 and 0.95 percentiles 

data['acousticness'].loc[data['acousticness'] > 1.017023]  = 1.017023
data['audio_valence'].loc[data['audio_valence'] < 0.048032]  = 0.048032
data['danceability'].loc[data['danceability'] < 0.094157]  = 0.094157
data['energy'].loc[data['energy'] < 0.094157]  = 0.094157
data['liveness'].loc[data['liveness'] > 0.891825]  = 0.891825
data['loudness'].loc[data['loudness'] < -26.502428]  = -26.502428
data['song_duration_ms'].loc[data['song_duration_ms'] > 390929.37000]  = 390929.37000
data['song_duration_ms'].loc[data['song_duration_ms'] < 54007.36300]  = 54007.36300
data['speechiness'].loc[data['speechiness'] > 0.465412]  = 0.465412
data['tempo'].loc[data['tempo'] > 204.984966]  = 204.984966
data['instrumentalness'].loc[data['instrumentalness'] > 0.987538]  = 0.987538
data['instrumentalness'].loc[data['instrumentalness'] < -0.004398]  = -0.004398


#Some Transformations to deal with the skewness of some features 
data['instrumentalness'] = np.log(1+data['instrumentalness'])
data['log_norm_EL'] = data['energy']+ np.exp(data['loudness'])*100
data['deNorm_acoust'] = data['acousticness']*data['log_norm_EL']
data['liveness_log'] = np.log(data['liveness'])
data['accousticness_sqrt'] = np.sqrt(data['acousticness'])
data['loudness_abs'] = abs(data['loudness'])
data['loudness_log'] = np.log(data['loudness_abs'])
data['speechiness_log'] = np.log(data['speechiness'])
data['tempo_log'] = np.log(data['tempo'])

# df['loudness'] = abs(df['loudness'])
# transformation(df, 'liveness', 'log')
# transformation(df, 'acousticness', 'sqrt')
# transformation(df, 'loudness', 'log')
# transformation(df, 'speechiness', 'log')
# transformation(df, 'tempo', 'log')

# df.drop('instrumentalness', axis=1, inplace=True)

data.isnull().sum()<jupyter_output><empty_output><jupyter_text>LABEL ENCODING<jupyter_code># from sklearn.preprocessing import LabelEncoder
# le = LabelEncoder()
# data["key"] = le.fit_transform(data["key"])
# data["time_signature"] = le.fit_transform(data["time_signature"])<jupyter_output><empty_output><jupyter_text>FREQUENCY ENCODING<jupyter_code>freq_columns = ['acousticness', 'audio_valence', 'danceability', 'energy',
       'instrumentalness', 'key', 'liveness', 'loudness', 'song_duration_ms',
       'speechiness', 'tempo','audio_mode','time_signature','key']
for col in freq_columns:
   data[col+'_freq'] = data[col].map(data[col].value_counts().to_dict())/len(data)
pd.options.display.max_rows = 2200
data.isnull().sum()<jupyter_output><empty_output><jupyter_text>ONE HOT ENCODING<jupyter_code># #Encode categorical features
# data = pd.get_dummies(data, columns=['audio_mode'])
# data.head()
data.describe().T<jupyter_output><empty_output><jupyter_text>FEATURE SELECTION<jupyter_code># Separate train and test data from the combined dataframe
train_df = data[:ntrain]
test_df = data[ntrain:]

# Check the shapes of the split dataset
train_df.shape, test_df.shape
# Select main columns to be used in training
main_cols = data.columns.difference(['id', 'song_popularity'])
X = train_df[main_cols]
y = train_df.song_popularity
# Create the model with several hyperparameters
# model = lgb.LGBMClassifier( n_estimators=1000, metric='AUC', learning_rate=0.04, subsample=0.7,
#                          colsample_bytree=0.8, num_leaves=64, reg_alpha=4, min_child_samples=5,subsample_freq=2)
model = cat(bootstrap_type='Bernoulli', 
                              iterations=1000, silent=True, eval_metric='AUC')
# model = xgb.XGBClassifier(learning_rate = 0.02859668002863679,
#     reg_lambda = 0.00012042589601632438,
#     reg_alpha = 16.683659682374774,
#     subsample = 0.4464941229852304,
#     colsample_bytree =0.5235019404878758,
#     max_depth = 4)
#
# Split into training and validation set
import random
random.seed(123)
#
train_features, valid_features, train_y, valid_y = train_test_split(X, y, test_size = 0.3, random_state = 47,stratify=y)
# Train using early stopping
model.fit(train_features, train_y, verbose = 200)
pd.options.display.max_rows = 2200
#define a function to create variable importance dataframe
def get_lgbm_varimp(model, train_columns, max_vars=300):
    
    # Scikit-learn API LGBMClassifier or LGBMRegressor was fitted, 
    # so using feature_importances_ property
    feature_importances_perc = (model.feature_importances_ / sum(model.feature_importances_)) * 100
    cv_varimp_df = pd.DataFrame([train_columns, feature_importances_perc]).T

    cv_varimp_df.columns = ['feature_name', 'varimp (%)']

    cv_varimp_df.sort_values(by='varimp (%)', ascending=False, inplace=True)

    cv_varimp_df = cv_varimp_df.iloc[0:max_vars]   

    return cv_varimp_df
#
get_lgbm_varimp(model, X.columns)
#drop irrelevant columns/variables from X features as identified in the feature selection step
columns = ["audio_valence_freq",
"instrumentalness_time_signature_mean_all",
"acousticness_freq",
"speechiness_freq",
"audio_mode",
"danceability_freq",
"liveness_freq",
"speechiness_audio_mode_mean_all",
"liveness_audio_mode_mean_all"]
#
X.drop(columns, inplace=True, axis=1)
#Dropping highly correlated attributes 
corr = X.corr()
# sns.heatmap(corr)
#
columns = np.full((corr.shape[0],), True, dtype=bool)
for i in range(corr.shape[0]):
    for j in range(i+1, corr.shape[0]):
        if corr.iloc[i,j] >= 0.9:
            if columns[j]:
                columns[j] = False
selected_columns = X.columns[columns]
X = X[selected_columns]
selected_columns
train_df = data[:ntrain].copy()
test_df= data[ntrain:].copy()
train_df.shape, test.shape
newtrain = train_df[selected_columns]
newtest = test_df[selected_columns]
newtrain['song_popularity'] = train_df['song_popularity']
newtest['song_popularity'] = test_df['song_popularity']
newtrain['id'] = train['id']
newtest['id'] = test['id']
newtrain.shape,newtest.shape
newtrain.isnull().sum()
#Dropping accousticness_sqrt since it has missing values 
newtrain.drop("accousticness_sqrt", inplace=True, axis=1)
newtest.drop("accousticness_sqrt", inplace=True, axis=1)
newtrain.shape,newtest.shape<jupyter_output><empty_output><jupyter_text>CREATING FOLDS<jupyter_code>import numpy as np
import pandas as pd
from sklearn import model_selection
newtrain['kfold'] = -1
kf = model_selection.KFold(n_splits=10, shuffle =True, random_state=999)
for fold, (train_indicies, valid_indicies) in enumerate(kf.split(X=newtrain)):
    newtrain.loc[valid_indicies, "kfold"] = fold
newtrain.to_csv("NewTrain.csv", index=False)
newtest.to_csv('NewTest.csv',index=False)
newtrain.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15162269805591702, 'fraction_numerical': 0.022634961946448994, 'mean_word_length': 8.258853596202993, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '18', 'dir': 'jupyter-structured-clean-dedup', 'id': '212391', 'license': 'Apache-2.0', 'n_tokens_mistral': 9622, 'n_tokens_neox': 9020, 'n_words': 1409, 'path': 'Song_Prediction-Folds-Preparation-Iterative imputer.ipynb', 'repo_name': 'koleshjr/SONG-POPULARITY-PREDICTION', 'size': '25359'}