<jupyter_start><jupyter_text>User Engagement Measure the difference of user engagement after a new UI design<jupyter_code>import os
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import seaborn as sns

from sklearn.metrics import r2_score
from scipy import stats
import statsmodels.stats.api as sms
%matplotlib inline
df_t1 = pd.read_csv('t1_user_active_min.csv')
df_t2 = pd.read_csv('t2_user_variant.csv')
df_t3 = pd.read_csv('t3_user_active_min_pre.csv')
df_t4 = pd.read_csv('t4_user_attributes.csv')
df_t1.head()
df_t1.describe()
df_t2.head()
df_t2.describe()
df_t2['variant_number'].plot()
print(len(df_t1))
# remove outliers

df_t1['z_score'] = np.abs(stats.zscore(df_t1['active_mins']))
df_t1 = df_t1[df_t1['z_score']<3]
print(len(df_t1))
df_t1['z_score'].plot()
sns.distplot(df_t1['z_score'])
# df1 groupby id
df_t1_grouped = df_t1.groupby(['uid'],as_index=False)
df_t1_sum = df_t1_grouped.sum()
df_t1_sum.head()
df_t1_sum.describe()
# df_t1_sum.dtypes
# df_t2.dtypes
# sns.distplot(df_t1_sum['active_mins'])<jupyter_output><empty_output><jupyter_text>t-test using table 1 & 2<jupyter_code># combine df_t1_sum with df_t2
df_t12 = pd.merge(df_t1_sum,df_t2,how = 'right',on = 'uid')
df_t12.head()
df_t12.shape
df_t12_nan = df_t12.dropna(subset=['active_mins'])
df_t12_nan.head()
df_t12_nan.shape
df_t12_nan[-5:]
X1_0 = df_t12_nan[df_t12_nan['variant_number']==0]['active_mins']
X1_1 = df_t12_nan[df_t12_nan['variant_number']==1]['active_mins']
plt.plot(X1_0)
plt.plot(X1_1)
print(X1_0.mean(),X1_1.mean())
stats.ttest_ind(X1_1,X1_0,equal_var = False,nan_policy='omit')
# confidence interval of t-test
cm1 = sms.CompareMeans(sms.DescrStatsW(X1_1), sms.DescrStatsW(X1_0))
cm1.tconfint_diff(alpha=0.05,alternative='two-sided',usevar='unequal')
cm1.ttest_ind(usevar='unequal')<jupyter_output><empty_output><jupyter_text>t-test using table 1, 2, & 3<jupyter_code>df_t3.head()
print(len(df_t3))
# delete outliers in table 3
df_t3['z_score'] = np.abs(stats.zscore(df_t3['active_mins']))
# remove outliers
df_t3 = df_t3[df_t3['z_score']<3]
print(len(df_t3))
df_t3['z_score'].plot()
# calculate sum
df_t3_grouped = df_t3.groupby(['uid'],as_index=False)
df_t3_sum = df_t3_grouped.sum()
df_t3_sum.head()
df_t3_sum = df_t3_sum.rename(columns = {"active_mins":"active_mins_pre","z_score":"z_score_pre"})
# combine table 2 and table 3
df_t123 = pd.merge(df_t12,df_t3_sum,how = 'left',on = 'uid')
df_t123.head()
print(len(df_t123))
df_t123_nan = df_t123.dropna(subset=['active_mins','active_mins_pre'])
print(len(df_t123_nan))
df_t123_nan[-5:]
X1_0 = df_t123_nan[df_t123_nan['variant_number']==0]['active_mins']
X1_1 = df_t123_nan[df_t123_nan['variant_number']==1]['active_mins']
X3_0 = df_t123_nan[df_t123_nan['variant_number']==0]['active_mins_pre']
X3_1 = df_t123_nan[df_t123_nan['variant_number']==1]['active_mins_pre']
print(len(X1_1),len(X3_1))
print(X1_0.mean(),X3_0.mean())
print(X1_1.mean(),X3_1.mean())<jupyter_output>460.4651391162029 295.81102018548825<jupyter_text>Compare pre- and post- treatment active minutes for controled group<jupyter_code># confidence interval of t-test 
cm31_0 = sms.CompareMeans(sms.DescrStatsW(X1_0), sms.DescrStatsW(X3_0))
cm31_0.tconfint_diff(alpha=0.05,alternative='two-sided',usevar='unequal')
cm31_0.ttest_ind(usevar='unequal')
stats.ttest_ind(X1_0,X3_0,equal_var = False,nan_policy='omit')<jupyter_output><empty_output><jupyter_text>Compare pre- and post- treatment active minutes for treatment group<jupyter_code>cm31_1 = sms.CompareMeans(sms.DescrStatsW(X1_1), sms.DescrStatsW(X3_1))
cm31_1.tconfint_diff(alpha=0.05,alternative='two-sided',usevar='unequal')
cm31_1.ttest_ind(usevar='unequal')
stats.ttest_ind(X1_1,X3_1,equal_var = False,nan_policy='omit')<jupyter_output><empty_output><jupyter_text>Deep Dive<jupyter_code># combine table 1,2,3,and 4
df_t1234 = pd.merge(df_t123,df_t4,how = 'left',on = 'uid')
df_t1234.head()
df_t4[df_t4['user_type']=='contributor'].count()
sns.countplot(x='gender',data=df_t1234)
sns.countplot(x='user_type',data=df_t1234)
sns.countplot(x='gender',hue = 'user_type',data=df_t1234)
# treated
df_t1234_t = df_t1234[df_t1234['variant_number']==1]
df_t1234_t.head()
sns.countplot(x='gender',data=df_t1234_t)
sns.countplot(x='user_type',data=df_t1234_t)
sns.countplot(x='gender',hue = 'user_type',data=df_t1234_t)
df_t1234_t.head()
df_t1234_t[df_t1234_t['user_type']=='contributor'].count()<jupyter_output><empty_output><jupyter_text>t-tests<jupyter_code>df_di = df_t1234.drop(columns = ['dt','signup_date','z_score','z_score_pre'])
df_di.head()
df_di = pd.get_dummies(df_di)
df_di.head()
df_di['user_type_contributor'].count()
df_di.dropna(subset=['active_mins','active_mins_pre'],inplace = True)
stats_di = []
print(stats_di[:2])<jupyter_output>[]<jupyter_text>Gender: male<jupyter_code>idx = (df_di['variant_number']==0) & (df_di['gender_male'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean of male without treatment: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([0,'male',-(X1.mean()-X2.mean()),p_value])
print(stats_di)
# treatment
idx = (df_di['variant_number']==1) & (df_di['gender_male'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean of male without treatment: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([1,'male',-(X1.mean()-X2.mean()),p_value])
print(stats_di)<jupyter_output>[[0, 'male', -46.803255214548244, 0.016467410365490737], [1, 'male', 192.74746291959406, 1.7600610401407062e-09]]<jupyter_text>Gender:Female<jupyter_code>idx = (df_di['variant_number']==0) & (df_di['gender_female'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([0,'female',-(X1.mean()-X2.mean()),p_value])
idx = (df_di['variant_number']==1) & (df_di['gender_female'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([1,'female',-(X1.mean()-X2.mean()),p_value])<jupyter_output>Pre & Post mean: 224.61574074074073, 355.52777777777777
p-value:1.2670504456974684e-05<jupyter_text>Gender:Unknown<jupyter_code>idx = (df_di['variant_number']==0) & (df_di['gender_unknown'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean controled group: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([0,'unknown',-(X1.mean()-X2.mean()),p_value])
idx = (df_di['variant_number']==1) & (df_di['gender_unknown'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean treated group: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([1,'unknown',-(X1.mean()-X2.mean()),p_value])<jupyter_output>Pre & Post mean treated group: 260.5369220151829, 386.2049689440994
p-value:0.006611247561988228<jupyter_text>User Type: Contributor<jupyter_code>idx = (df_di['variant_number']==0) & (df_di['user_type_contributor'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean controled group: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([0,'Contributor',-(X1.mean()-X2.mean()),p_value])
idx = (df_di['variant_number']==1) & (df_di['user_type_contributor'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean controled group: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([1,'Contributor',-(X1.mean()-X2.mean()),p_value])<jupyter_output>Pre & Post mean controled group: 3231.299212598425, 4708.031496062992
p-value:0.06580880513818083<jupyter_text>User Type: New User<jupyter_code>idx = (df_di['variant_number']==0) & (df_di['user_type_new_user'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean controled group: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([0,'New User',-(X1.mean()-X2.mean()),p_value])
idx = (df_di['variant_number']==1) & (df_di['user_type_new_user'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean controled group: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([1,'New User',-(X1.mean()-X2.mean()),p_value])<jupyter_output>Pre & Post mean controled group: 6.487012987012987, 54.62337662337662
p-value:5.585057715892592e-10<jupyter_text>User Type: Non-Reader<jupyter_code>idx = (df_di['variant_number']==0) & (df_di['user_type_non_reader'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean controled group: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([0,'Non-Reader',-(X1.mean()-X2.mean()),p_value])
idx = (df_di['variant_number']==1) & (df_di['user_type_non_reader'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean controled group: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([1,'Non-Reader',-(X1.mean()-X2.mean()),p_value])<jupyter_output>Pre & Post mean controled group: 99.44206621004567, 164.45462328767124
p-value:6.108660456222139e-29<jupyter_text>User Type: Reader<jupyter_code>idx = (df_di['variant_number']==0) & (df_di['user_type_reader'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean controled group: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([0,'Reader',-(X1.mean()-X2.mean()),p_value])
idx = (df_di['variant_number']==1) & (df_di['user_type_reader'] == 1)
X1 = df_di[idx]['active_mins_pre']
X2 = df_di[idx]['active_mins']
print("Pre & Post mean controled group: {}, {}".format(X1.mean(),X2.mean()))
t_stat,p_value = stats.ttest_ind(X1,X2,equal_var = False,nan_policy='omit')
print("p-value:{}".format(p_value))
stats_di.append([1,'Reader',-(X1.mean()-X2.mean()),p_value])
df_stats = pd.DataFrame(stats_di,columns = ['variant_number','group','diff_mean','p_value'])
df_stats['p_sign'] = [1 if i < 0.01 else 0 for i in df_stats['p_value']]
fig, (ax1,ax2) = plt.subplots(2,1,figsize=(8,6))
sns.barplot(x = 'group',y='diff_mean',hue = 'variant_number',data = df_stats,ax =ax1)
ax1.set_xlabel('')
ax1.grid(axis = 'y')
ax1.legend(loc='center left', bbox_to_anchor=(1, 0.5))
# ax1.set_yscale('log')
sns.catplot(x = 'group',y='p_sign',hue = 'variant_number',data = df_stats,ax =ax2)
ax2.set_ylabel('p-significance')
ax2.set_xlabel('')
ax2.set_yticks([0,1])
ax2.legend(loc='center left', bbox_to_anchor=(1, 0.5))
# fig, ax = plt.subplots(figsize=(10,5))
# sns.barplot(x = 'group',y='p_value',hue = 'variant_number',data = df_stats,ax =ax)
# fig, ax = plt.subplots(figsize=(8,5))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20154780727302987, 'fraction_numerical': 0.0711492052925023, 'mean_word_length': 9.5513608428446, 'pattern_counts': {'":': 2, '<': 52, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '323397', 'license': 'MIT', 'n_tokens_mistral': 5686, 'n_tokens_neox': 5279, 'n_words': 611, 'path': 'User Engagement.ipynb', 'repo_name': 'yaotang1208/User-Engagement', 'size': '12017'}