<jupyter_start><jupyter_text>Exploratory Data Analysis  1. More feature engineering and selection<jupyter_code>import numpy as np
import pandas as pd
feature_pickle = 'C:\\Users\\Sean\\Documents\\BitTiger\\Capston_music_player_python\\features_and_label.pkl'
df = pd.read_pickle(feature_pickle)
df.head()
log_pickle = 'C:\\Users\\Sean\\Documents\\BitTiger\\Capston_music_player_python\\reduced_play_log.pkl'
df_play_log = pd.read_pickle(log_pickle)
df_play_log.head()
import matplotlib.pyplot as plt

% matplotlib inline

plt.style.use('ggplot')
plt.rcParams["figure.figsize"] = (8,6)
ax0 = df_play_log.groupby('date').size().unstack().fillna(0).plot.bar()
plt.show()
plt.rcParams["figure.figsize"] = (8,6)
ax = df_play_log.groupby(['date','label']).size().unstack().fillna(0).plot.bar(color = ['b','r'])
plt.show()<jupyter_output><empty_output><jupyter_text>I. Completion ratio: play_time / song_length<jupyter_code># completion ratio of each song playing activity
df_play_log['play_ratio'] = df_play_log.play_time / df_play_log.song_length
df_play_log['play_ratio'] = df_play_log['play_ratio'].apply(abs)
df_play_log.head()
df_play_log.loc[(df_play_log.song_length == 0) |(df_play_log.song_length == 0.0), 'play_ratio'] = 1
df_play_log.head()
df_play_log.play_ratio.describe()
df_play_log.loc[df_play_log.play_ratio > 1,'play_ratio'] = 1
# a proof that defining complete ratio = 1 is valid for song_length ==0
df_play_log.loc[(df_play_log.song_length==0) | (df_play_log.song_length==0.0)].label.mean()
# in the play log level, replay over 1000 times is a clear sign of churn
df_play_log.loc[(df_play_log.play_ratio<1)].label.mean(),df_play_log.loc[(df_play_log.play_ratio==1)].label.mean()
# in the user level
df['avg_complete_ratio'] = list(df_play_log.groupby('uid').mean()['play_ratio'])
df.head()
df['avg_complete_ratio'].describe()
df.loc[df.avg_complete_ratio<0.1].label.mean(), df.loc[df.avg_complete_ratio>=0.1].label.mean()
df.loc[df.avg_complete_ratio<=0.1].shape<jupyter_output><empty_output><jupyter_text>II. ratio_n_over_23： count_play_n / count_play_23        This indicates how active a user is near the cutoff date<jupyter_code># creation of new column names
columns = ['count_play_1','count_play_3','count_play_7','count_play_14','count_play_23']
for i in range(1,len(columns)):
    for j in range(i):
        print 'ratio_%s_over_%s' % (columns[j].split('_')[-1],columns[i].split('_')[-1])
for i in range(1,len(columns)):
    for j in range(i):
        new_col =  'ratio_%s_over_%s' % (columns[j].split('_')[-1],columns[i].split('_')[-1])
        df[new_col] = df[columns[j]].astype(float) / (df[columns[i]] + 0.00001)

df.head()
# a quick check of churn rate
df.loc[:,['ratio_1_over_3','ratio_1_over_7','ratio_3_over_7','ratio_1_over_14',\
       'ratio_3_over_14','ratio_7_over_14','ratio_1_over_23','ratio_3_over_23',\
       'ratio_7_over_23','ratio_14_over_23']].describe()
df.groupby('label').mean().loc[:,['ratio_1_over_3','ratio_1_over_7','ratio_3_over_7','ratio_1_over_14',\
       'ratio_3_over_14','ratio_7_over_14','ratio_1_over_23','ratio_3_over_23',\
       'ratio_7_over_23','ratio_14_over_23']]<jupyter_output><empty_output><jupyter_text>III. active_days: days when a user had at least one activity        This might take off the influence of the outliers, who play huge amount of songs each day<jupyter_code>df_daily_play = df_play_log.groupby(['uid','date']).size().unstack().fillna(0)
df_daily_play.head()
# if a user had any activity on the days
df_play_log.groupby(['uid','date']).size().unstack().notnull().head()
df['active_days']= list(df_play_log.groupby(['uid','date']).size().unstack().notnull().sum(axis=1))
df.head()
df.groupby(['active_days','label']).size().unstack()
df.loc[df.active_days > 8].label.mean(),df.loc[df.active_days <= 8].label.mean()<jupyter_output><empty_output><jupyter_text>Days since last active    This is actually the first feature that needs to be figure out for this churn model.        The median or a certain quantile should be used for defining churns.<jupyter_code>df_daily_play = df_play_log.groupby(['uid','date']).size().unstack().notnull()
df_daily_play.head()
integer23 = range(1,24)
active_day = df_daily_play.iloc[1,:]*integer23
active_day, active_day.index
active_day.index[max(active_day)-1]
24 - max(active_day)
df_daily_play['days_since_last_play'] = \
    [24-max(df_daily_play.iloc[i,:]*integer23) for i in xrange(df_daily_play.shape[0])]
df_daily_play.head()
df_daily_play['days_since_last_play'].value_counts()
df['days_since_last_play'] = list(df_daily_play['days_since_last_play'])
df.head()
df.days_since_last_play.describe()
# average churn rate vs. median of the days_since_last_play

df.loc[df.days_since_last_play > 3].label.mean(),df.loc[df.days_since_last_play <= 3].label.mean()
df_features = df.drop(['count_play_1','count_play_3','count_play_7','count_play_14','label'],axis = 1)

df_features['label'] = df.label
df_features.head()
df_features.shape
df_features.iloc[0,:].apply(type).value_counts()<jupyter_output><empty_output><jupyter_text>EDA1. For categorical features, or numerical features with discrete values: *Bar plot*2. For numeric features with continues values: *histogram*     1. device vs. churn<jupyter_code>df_device = df_features.groupby(['device','label']).size().unstack()
df_device
import matplotlib.pyplot as plt

% matplotlib inline

plt.style.use('ggplot')
ax = df_device.plot.bar()
plt.show()<jupyter_output><empty_output><jupyter_text>2. Song_type vs. Churn<jupyter_code>df_song_type = df_features.groupby(['major_song_type','label']).size().unstack()
df_song_type
ax = df_song_type.plot.bar()
plt.show()<jupyter_output><empty_output><jupyter_text>3. active_days<jupyter_code>df_active_days = df_features.groupby(['active_days','label']).size().unstack().fillna(0)
df_active_days
ax = df_active_days.plot.bar()
plt.xlim(2.5,22.5)
plt.rcParams["figure.figsize"] = (8,4)
plt.show()<jupyter_output><empty_output><jupyter_text>4. days_since_last_play<jupyter_code>df_days_since_last_play = df_features.groupby(['days_since_last_play','label']).size().unstack()
df_days_since_last_play
ax = df_days_since_last_play.plot.bar()
plt.show()<jupyter_output><empty_output><jupyter_text>5. total_play_time<jupyter_code>df_features.groupby('label').mean()
df_features.total_play_time.describe()
df_play_time = df_features.loc[:,['total_play_time','label']]
df_play_time.head()
df_play_time['bins'] = list(pd.cut(df_features.total_play_time, bins=[0,200,1000,2000], 
       right=False, labels=None, retbins=False, precision=1, include_lowest=True))
df_play_time.bins = df_play_time.bins.fillna('[2000,inf)')
df_play_time.head()
#df_play_time.loc[df_play_time.bins.isnull(),'label'].mean()
df_play_time_bins = df_play_time.groupby(['bins','label']).size().unstack().fillna(0)
df_play_time_bins.head()
df_play_time_bins
ax = df_play_time_bins.plot.bar()
plt.rcParams["figure.figsize"] = (6,4)
plt.show()<jupyter_output><empty_output><jupyter_text>6. ratios of accelerations<jupyter_code>df_features.columns
import fnmatch

ratios = fnmatch.filter(df_features.columns, 'ratio*')
ratios
df_ratios = df_features.loc[:,ratios]
df_ratios.head()
df_ratios.describe()
pd.cut(df_ratios.ratio_1_over_23, bins=np.linspace(0,0.3,61), 
       right=False, labels=None, retbins=False, precision=1, include_lowest=True)
df_1_over_23 = df_features.loc[:,['ratio_1_over_23','label']]
df_1_over_23['bins'] = list(pd.cut(df_ratios.ratio_1_over_23, bins=np.linspace(0.0,0.301,61), 
       right=False, labels=None, retbins=False, precision=1, include_lowest=True))
df_1_over_23.head()
df_1_bins = df_1_over_23.groupby(['bins','label']).size().unstack().fillna(0)
df_1_bins.head()
ax = df_1_bins.plot.bar()
plt.rcParams["figure.figsize"] = (10,4)
plt.xlim(-0.5,5.5)
plt.show()
df_3_over_23 = df_features.loc[:,['ratio_3_over_23','label']]
df_3_over_23['bins'] = list(pd.cut(df_ratios.ratio_3_over_23, bins=np.linspace(0.00,0.5,101), 
       right=False, labels=None, retbins=False, precision=1, include_lowest=True))
df_3_bins = df_3_over_23.groupby(['bins','label']).size().unstack().fillna(0)
ax = df_3_bins.plot.bar()
#plt.rcParams["figure.figsize"] = (10,4)
plt.xlim(-0.5,5.5)
plt.show()
df_7_over_23 = df_features.loc[:,['ratio_7_over_23','label']]
df_7_over_23['bins'] = list(pd.cut(df_ratios.ratio_7_over_23, bins=np.linspace(0.00,0.7,71), 
       right=False, labels=None, retbins=False, precision=1, include_lowest=True))
df_7_bins = df_7_over_23.groupby(['bins','label']).size().unstack().fillna(0)
ax = df_7_bins.plot.bar()
#plt.rcParams["figure.figsize"] = (12,6)
plt.xlim(-0.5,5.5)
plt.show()
df_ratios['label']=df_features.label
df_ratios.loc[df_ratios.label == 0].describe()
df_ratios.loc[df_ratios.label == 1].describe()
df_features['play_time_bins'] = df_play_time.bins
df_features.head()
df_features = df_features.drop('play_time_label',axis = 1)
df_features.head()
filename_pickle = 'C:\\Users\\Sean\\Documents\\BitTiger\\Capston_music_player_python\\modified_features_and_label.pkl'
df_features.to_pickle(filename_pickle)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16874379618396382, 'fraction_numerical': 0.032204698356678066, 'mean_word_length': 7.644423260247855, 'pattern_counts': {'":': 0, '<': 50, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '544634', 'license': 'MIT', 'n_tokens_mistral': 3743, 'n_tokens_neox': 3543, 'n_words': 466, 'path': '.ipynb_checkpoints/3. Exploratory Data Analysis-checkpoint.ipynb', 'repo_name': 'YXChen512/Churn_Analysis_music_player_app', 'size': '9067'}