<jupyter_start><jupyter_text>Special thanks to https://www.kaggle.com/somang1418/tuning-hyperparameters-under-10-minutes-lgbmdef reduce_mem_usage(df, verbose=True):    numerics = ['int8','int16', 'int32', 'int64', 'float16', 'float32', 'float64']    start_mem = df.memory_usage().sum() / 1024**2        for col in df.columns:        col_type = df[col].dtypes        if col_type in numerics:            c_min = df[col].min()            c_max = df[col].max()            if str(col_type)[:3] == 'int':                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:                    df[col] = df[col].astype(np.int8)                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:                    df[col] = df[col].astype(np.int16)                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:                    df[col] = df[col].astype(np.int32)                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:                    df[col] = df[col].astype(np.int64)              else:                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:                    df[col] = df[col].astype(np.float16)                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:                    df[col] = df[col].astype(np.float32)                else:                    df[col] = df[col].astype(np.float64)        end_mem = df.memory_usage().sum() / 1024**2    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))    return df df_reduce = reduce_mem_usage(df)X_train, X_test, y_train, y_test = train_test_split(df_reduce[features], df_reduce[label], test_size=0.3, random_state=6)y_train = pd.DataFrame(y_train)train = X_train.merge(y_train, left_index = True, right_index = True)y_test = pd.DataFrame(y_test)test = X_test.merge(y_test, left_index = True, right_index = True)<jupyter_code>df['apt_date2'] = df.apply(lambda x: x.apt_date.strftime("%x"), axis = 1)
scheduled_days = df.groupby(['apt_date2'])[['apt_date']].count()
scheduled_days.head()
scheduled_days.columns = ['i', 'index', 'date', 'count']
scheduled_days['date'] = pd.to_datetime(scheduled_days['date'])

print('first scheduled: ' + str(scheduled_days.date.min()))
print('most recent scheduled: ' + str(scheduled_days.date.max()))

sns.scatterplot(x = 'date', y = 'count', data = scheduled_days)
plt.title('Number of Appointments per Scheduled Day')
plt.xlabel('Scheduled Day')
plt.xlim('2017-03', '2021-12')
plt.gcf().set_size_inches(10, 6)
plt.show()
herp = df.apt_type[1:100000].value_counts()[1:40]
#print(herp.count)
#plt.xticks(rotation='vertical')
#plt.bar(herp.index,herp)

plt.pie(herp,labels=herp.index)
plt.show()
print('Total appointments: ' + format(df.shape[0], ",d"))
print('Distinct patients: ' + format(df.pat_id.unique().shape[0], ",d"))
print('Patients with more than one appointment: ' + format((df['pat_id'].value_counts() > 1).sum(), ",d"))
df.age.describe()
sns.set()

def cat_var(df3, var):
    
    print(df3.groupby([var])['response'].mean())
    
    ns_rate = [df3.groupby([var])['response'].mean()[i] for i in df3[var].unique()]
    s_rate = [1-df3.groupby([var])['response'].mean()[i] for i in df3[var].unique()]
    barWidth = 5

    plt.bar(df3[var].unique(), ns_rate, color='lightcoral', edgecolor='white', width=barWidth, label = 'response')
    plt.bar(df3[var].unique(), s_rate, bottom=ns_rate, color='mediumseagreen', edgecolor='white', width=barWidth, label = 'Show')
    plt.axhline(y=df3['response'].mean(), color='black', linewidth= 0.8, linestyle='--', label = 'Overall mean')
    plt.xticks(df3[var].unique())
    plt.xlabel(var)
    plt.legend(loc='upper left', bbox_to_anchor=(1,1), ncol=1)
    plt.title('response Rate by '+ var, fontsize=15)
    plt.gcf().set_size_inches(6, 6)
    plt.show() 
    
    counts = np.array(df3.groupby([var])['response'].sum())
    nobs = np.array(df3.groupby([var])['response'].count())

    table = df3.groupby(['response', var]).size().unstack(var)
    pvalue = ss.chi2_contingency(table.fillna(0))[1]
    
    print('Means test p-value: {:1.3f}'.format(pvalue))
    if pvalue < 0.05:
        print('Reject null hypothesis: no-show rate is different for at least one group')
    else:
        print('Cannot reject no-show rates are same for all groups')
df_net = df[(df['net_hour'] < -1) & (df['net_hour'] > -100)]
cat_var(df_net,'age')
print('Correlation with No-Show: %.3f' % ss.pointbiserialr(df['response'], df['net_hour'])[0])

df[df.response == 1].count<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18578178694158076, 'fraction_numerical': 0.030927835051546393, 'mean_word_length': 3.671013039117352, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '26228', 'license': 'MIT', 'n_tokens_mistral': 1808, 'n_tokens_neox': 1705, 'n_words': 299, 'path': 'exploration.ipynb', 'repo_name': 'carolkng/easymarkit-2019', 'size': '4656'}