<jupyter_start><jupyter_text>Sleep research: Programming Challenge<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pylab as plt
import seaborn as sns
from pathlib import Path
#Set data path
sleep_deprived_data_path = Path('../data/sleep_deprived_coding.csv')
df = pd.read_csv(sleep_deprived_data_path)
df.head()
df.describe()<jupyter_output><empty_output><jupyter_text>Data Exploration<jupyter_code>#Visualize the distribution using normal/deprived as hue
sns.pairplot(df, hue='sleep')
#Create function to generate aggregated table grouped by normal and deprived
def aggregate_sleep_groups(dataf):
    
    agg_dataf = (dataf
                 .groupby('sleep')
                 .agg(n=('id', 'count'),
                      mean_gpa=('gpa', np.mean),
                      mean_unit_tests=('passed_unit_tests', np.mean),
                      mean_asserts=('passed_asserts', np.mean),
                      mean_user_stories=('tackled_user_stories', np.mean))
                )
    return agg_dataf
#Use aggregated function to show means from both groups
agg_dataf = df.pipe(aggregate_sleep_groups)
agg_dataf
#Visualize the difference using barplots
fig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(figsize=(10,8),nrows=2, ncols=2, sharex=False, sharey=False)

for ax, column in zip(fig.axes, agg_dataf.columns[1:]):
    sns.barplot(x=agg_dataf.index, y=column, data=agg_dataf, ax=ax)<jupyter_output><empty_output><jupyter_text>SamplingIn order to check whether the observed coding performance difference between the sleep and deprived group is due to chance, we are going to take a lot of random samples and check the mean distribution.<jupyter_code>#Two functions to take random samples and calculate the difference.

#Function to shuffle id's and assign deprived (15) versus normal (21) randomly.
def reshuffle(dataf):
    
    shuffled_dataf = (dataf
                      .sample(dataf['id'].count())
                      .reset_index(drop=True)
                      .assign(sleep=lambda d: np.where(d.index < 15, 'deprived', 'normal'))
                     )
    
    return shuffled_dataf

#Function to calculate difference between deprived and normal group
def calc_diff_sleep(dataf):
    
    agg_dataf = (dataf
                        .groupby('sleep')
                        .agg(mean_unit_tests=('passed_unit_tests', np.mean),
                             mean_asserts=('passed_asserts', np.mean),
                             mean_user_stories=('tackled_user_stories', np.mean))
                       ).T
    
    difference_dataf = agg_dataf['deprived'] - agg_dataf['normal'] 
    
    return difference_dataf
#Function to store the differences between deprived and normal group from samples, take 1000 samples by default
def shuffle_simulation(dataf, n=1000):
    
    results = np.zeros((n, 3))
    
    for i in range(n):
        results[i, :] = calc_diff_sleep(reshuffle(dataf))
        
    results_difference_dataf = pd.DataFrame(results,
                                            columns=['diff_unit_tests',
                                                     'diff_asserts',
                                                     'diff_user_stories'])
    return results_difference_dataf
sample_dataf_diff = df.pipe(shuffle_simulation)
sample_dataf_diff.head()<jupyter_output><empty_output><jupyter_text>Analyze sampling distribution  Check for differences between sleep and deprived group<jupyter_code>#Create function to plot the group difference in sample distributions
#Draw a line to illustrate the difference in the original dataset
def plot_diff(dataf):
    
    plt.figure(figsize=(16,4))

    dict_diff = dict(dataf)

    for idx, name in enumerate(['unit_tests', 'asserts', 'user_stories']):
        plt.subplot(131 + idx)
        plt.hist(sample_dataf_diff[f'diff_{name}'])
        plt.vlines(dict_diff[f'mean_{name}'], 0, 300, color='red')
        plt.title(name)
df.pipe(calc_diff_sleep).pipe(plot_diff_sleep)
#Function to return a dataframe with the probabilities to find a difference as big as we found due to chance.
#For the sleep groups
def chance_diff_sleep(sample_dataf, original_dataf):
    
    dict_diff = dict(calc_diff_sleep(original_dataf))
    
    chance_results = {}
    
    for column, value in zip(sample_dataf.columns, dict_diff.values()):
        chance_results[column] = sample_dataf.assign(limit=lambda d: d[column] < value)['limit'].mean()

    chance_dataf = pd.DataFrame(chance_results.values(),
                                index=chance_results.keys(),
                                columns=['chance_diff'])
    
    return chance_dataf    
sample_dataf_diff.pipe(chance_diff_sleep, original_dataf=df)<jupyter_output><empty_output><jupyter_text>Check for differences between different levels of GPA<jupyter_code>#Function to calculate difference between high and low GPA group
def calc_diff_gpa(dataf):
    
    agg_dataf = (dataf
                        .assign(gpa=lambda d: d['gpa'] < d['gpa'].mean())
                        .groupby('gpa')
                        .agg(mean_unit_tests=('passed_unit_tests', np.mean),
                             mean_asserts=('passed_asserts', np.mean),
                             mean_user_stories=('tackled_user_stories', np.mean))
                       ).T
    
    difference_dataf = agg_dataf[False] - agg_dataf[True] 
    
    return difference_dataf
df.pipe(calc_diff_gpa).pipe(plot_diff)
#Function to return a dataframe with the probabilities to find a difference as big as we found due to chance.
#For the GPA groups
def chance_diff_gpa(sample_dataf, original_dataf):
    
    dict_diff = dict(calc_diff_gpa(original_dataf))
    
    chance_results = {}
    
    for column, value in zip(sample_dataf.columns, dict_diff.values()):
        chance_results[column] = sample_dataf.assign(limit=lambda d: d[column] < value)['limit'].mean()

    chance_dataf = pd.DataFrame(chance_results.values(),
                                index=chance_results.keys(),
                                columns=['chance_diff'])

    return chance_dataf  
sample_dataf_diff.pipe(chance_diff_gpa, original_dataf=df)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09583265769417869, 'fraction_numerical': 0.0056753688989784334, 'mean_word_length': 2.8215613382899627, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '411186', 'license': 'MIT', 'n_tokens_mistral': 1919, 'n_tokens_neox': 1825, 'n_words': 426, 'path': 'notebooks/sleep_deprived_research.ipynb', 'repo_name': 'casperdamen123/python_projects_DiP', 'size': '6167'}