<jupyter_start><jupyter_text>GBE> This submodule handles the analysis of GBE data.  It is build around GBEProvider which is inherited (copies all functions) from `DataProvider`.<jupyter_code>#hide
from nbdev.showdoc import *
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
import seaborn as sns
#export
import numpy as np
from trr265.data_provider import DataProvider, get_efficiently
from fastcore.foundation import patch
import pandas as pd
from matplotlib import pyplot as plt
import re
#export
class GBEProvider(DataProvider):
    '''This class builds upon DataProvider and focusses on GBE data.'''
    def __init__(self, data_folder_path):
        DataProvider.__init__(self, data_folder_path)
        self.GBE_COLUMNS = ['FruitTapGame','WorkingMemoryGame','CardGame','RewardAndHappinessGame']<jupyter_output><empty_output><jupyter_text>Getting GBE Data > We first select GBE data from movisense.<jupyter_code>#export 
@patch
def get_gbe_from_movisense(self:GBEProvider):
    """
    This function pulls GBE data from movisense and sets cancelled sessions to None.  Here, we only include rows that have at least one non-cancelled GBE game.
    """
    # Getting GBE data
    df = self.get_mov_data().query("Form == 'GreatBrainExperiment'")
    # Setting cancelled sessions to missing
    df[self.GBE_COLUMNS] = df[self.GBE_COLUMNS].replace({'{canceled": true}"':None})
    # Only included rows with at least one non-cancelled GBE game
    df = df[~df[self.GBE_COLUMNS].isna().all(axis=1)]
    df = df.sort_values(['participant','trigger_date'])
    shifted = df.groupby('participant').sampling_day.shift(1)
    df['time_since_last_gbe'] = df['sampling_day'] - shifted
    df['time_since_last_gbe'].fillna(0, inplace = True)
    df['session_number'] = df.groupby('participant').cumcount() + 1
    # Creating the trigger type variable
    replace_dict = {
    'Button Pressed Spiele starten und Initialfragen (~40min)':'initial',
    'Participant Defined Time Trigger':'ema',
    'Button Pressed Bedarfstart GBE (Ausnahme)':'optional',
    'Repeated':'optional',
    'Button Pressed Bedarfsstart GBE':'optional'}
    df['trigger_type'] = df.Trigger.apply(lambda x: x.replace('Pressed:','Pressed').split(':')[0]).replace(replace_dict)
    df['gbe_index'] = df.participant + '_' + df.session_number.apply(lambda x: '%03d'%int(x))
    df = df.set_index('gbe_index')
    return df<jupyter_output><empty_output><jupyter_text>Defining the initial dataset> We sort the data by starting date and first trigger date (in case participants have same starting date).  Based on this sorted dataset, we select the first 300 participants.<jupyter_code>#export
@patch
def define_initial_dataset(self:GBEProvider, df):
    initial_pps = df.groupby('participant').first().sort_values(['starting_date','trigger_date']).iloc[:300].index
    df['is_initial'] = df.participant.isin(initial_pps)
    df = df.sort_index()
    return df
dp = GBEProvider('/Users/hilmarzech/Projects/trr265/trr265/data/')
df = dp.get_gbe_from_movisense()
df = dp.define_initial_dataset(df)
df.groupby('participant')[['is_initial','starting_date','trigger_date']].first()
df.query('is_initial==True').starting_date.describe()
df.query('is_initial==False').starting_date.describe()<jupyter_output><empty_output><jupyter_text>Defining the baseline sessionsBaseline sessions are the participants first two sessions if completed on the same day.  Otherwise the participants first session if it was not triggered by an EMA trigger.<jupyter_code>#export
@patch
def define_baseline_sessions(self:GBEProvider, df):
    first_two_sessions = df.session_number <= 2
    same_day = df.time_since_last_gbe == 0
    initial_or_optional = df.trigger_type != 'ema'
    before_ema = df.sampling_day < 22 # Ema starts on sampling day 22, so we only consider baseline sessions that happened before that
    df['is_baseline'] = first_two_sessions & same_day & initial_or_optional & before_ema
    return df
df = dp.define_baseline_sessions(df)
df[['participant','session_number','trigger_type','time_since_last_gbe','is_baseline']]<jupyter_output><empty_output><jupyter_text>Checks  Number of baseline sessions per participant<jupyter_code>check = df.groupby('participant').is_baseline.sum().value_counts().to_frame()
check.columns = ['Participants']
check.index.rename('Baseline sessions', inplace = True)
check<jupyter_output><empty_output><jupyter_text>> Most participants have two baseline sessions.  However, some only have one and a few participants have no baseline session.  Time between baseline sessions<jupyter_code>#export 
@patch
def find_gbe_game_starting_time(self:GBEProvider, s):
    '''As baseline GBEs have the same trigger date, we first have to parse the time from the GBE data.'''
    nan_time = pd.to_datetime(np.nan)
    try:
        gbe_columns = s.split(',')
    
    except:
        return nan_time
    if 'timestarted' not in gbe_columns:
        return nan_time
    time_started_index = gbe_columns.index('timestarted')
    time_started = pd.to_datetime(s.split('\n')[1].split(',')[time_started_index][2:-2])
    return time_started



@patch
def check_time_between_sessions(self:GBEProvider, df): 
    time_1 = pd.to_datetime(np.nan)
    time_2 = pd.to_datetime(np.nan)
    if len(df)==2:
        time_1 = self.find_gbe_game_starting_time(df.iloc[0])
        time_2 = self.find_gbe_game_starting_time(df.iloc[1])
    try:
        return (time_2 - time_1).seconds/60
    except:
        return np.nan
time_between_sessions = df.query('is_baseline').groupby('participant').FruitTapGame.agg(dp.check_time_between_sessions)
sns.distplot(time_between_sessions, label = "Time in minutes between baseline sessions")
plt.legend()
sns.despine()
sns.distplot(time_between_sessions[time_between_sessions < 200], label = "Time in minutes between baseline sessions (zoomed in)")
plt.legend()
sns.despine()<jupyter_output><empty_output><jupyter_text>> Most participants took under one hour between baseline sessions.  Note that there is some variance, so we could check whether reliability already decreases within minutes.  Time between EMA sessions<jupyter_code>sns.distplot(df.query("is_baseline==False").time_since_last_gbe, label = "Time in minutes between ema sessions")
sns.despine()
sns.distplot(df.query("(is_baseline==False) and (time_since_last_gbe < 60)").time_since_last_gbe, label = "Time in minutes between ema sessions")
sns.despine()<jupyter_output><empty_output><jupyter_text>> Most ema sessions are completed after 28 days (as expected).  However, some ema sessions seem to have been completed right after the baseline sessions.  Perhaps these sessions should be removed from the data. > Warning: Some ema sessions are completed right after the baseline sessions.  Finding EMA sessionsA difficulty in getting the GBE data is that participants could complete optional sessions.  Some of these sessions fall into the protocol (one session every 28 days), other sessions do not (e.g. several EMA-sessions completed on the same day).  The below determine whether EMA sessions were completed according to protocol and removes all non-protocol sessions.<jupyter_code>#export
def determine_ema_session_numbers(_df):
    '''Ema sessions start the earliest after sampling day 22.  The following year is split into 28-day intervals, each corresponding to one EMA session.'''
    _df['ema_session'] = pd.cut(_df.sampling_day, np.arange(21, 392, 28), labels = [x for x in range(1,14)]).astype(float)
    return _df
    
def check_repeating_sessions(_df):
    '''Sometimes EMA sessions end up happening in the same 28 days interval.  In these cases, participants therefore have repeating sessions.  If a repeating session is one the same day, it should certainly be removed.  However, if a repeating session happens far enough from other sessions and the subsequent session is missing, we can assume that this session is actually a real session.  Although, the nominal distance between sessions is 28 days, participants can postpone sessions, thus reducing the time to the next session.  Specifying at least 14 one days between sessions, therefore seems reasonable.'''
    _df['ema_session_repeating'] = ((_df.ema_session.astype(float) - _df.ema_session.astype(float).shift(1))==0)
    _df['total_ema_sessions_completed'] = _df.groupby('participant').ema_session.transform(lambda x:len(x.dropna().unique()))
    _df['time_since_last_session'] = ((_df.sampling_day - _df.groupby('participant').sampling_day.shift(1)))
    _df['time_to_next_session'] = ((_df.sampling_day - _df.groupby('participant').sampling_day.shift(-1))*-1)
    # Deciding wether to increase session number
    sessions_missing = _df['total_ema_sessions_completed'] < 13
    window = 14
    enough_distance = (_df.time_since_last_session>window) & (_df.time_to_next_session>window)
    _df['increase_session_number'] = (_df.ema_session_repeating & enough_distance & sessions_missing).astype(int)
    return _df

def increase_session_numbers(_df):
    _df['ema_session'] = _df.ema_session + _df.increase_session_number
    return(_df)

def filter_repeating_sessions(_df):
    _df = _df.query('ema_session_repeating==False')
    _df = _df.query('is_baseline or (ema_session.isna() == False)')
    return _df

@patch
def fix_repeating_ema_sessions(self:GBEProvider, _df):
    _df = determine_ema_session_numbers(_df)
    old_df = _df
    for i in range(13):
        _df = check_repeating_sessions(_df)
        _df = increase_session_numbers(_df)
    old_sessions = _df.index
    
    _df = filter_repeating_sessions(_df)
    new_sessions = _df.index
    removed_sessions = list((set(old_sessions)-set(new_sessions)))
    #return old_df.loc[removed_sessions]
    removed_participants = list(set([s.split('_')[0] for s in removed_sessions]))
    _df = filter_repeating_sessions(_df)
    _df = check_repeating_sessions(_df)
    print("%d participants had unplanned sessions.  %d sessions (%.2f%%) were removed."%(len(removed_participants), len(removed_sessions), len(old_df)/len(removed_sessions)))
    _df['session_number_ex_protocol'] = _df.session_number
    _df['session_number'] = (_df.ema_session+2).fillna(_df.session_number)
    return _df<jupyter_output><empty_output><jupyter_text>Getting the final data<jupyter_code>#export
@patch
@get_efficiently
def get_gbe_data(self:GBEProvider):
    df = self.get_gbe_from_movisense()
    df = self.define_initial_dataset(df)
    print(df.is_initial.isna().mean())
    df = self.define_baseline_sessions(df)
    print(df.is_initial.isna().mean())
    df = self.fix_repeating_ema_sessions(df)
    print(df.is_initial.isna().mean())
    df = df.reset_index()
    df['gbe_index'] = df.participant + '_' + df.session_number.apply(lambda x: '%03d'%int(x))
    df = df.set_index('gbe_index')
    return df
#export
@patch
def get_initial_baseline(self:GBEProvider):
    df = self.get_gbe_data()
    return df.query("is_initial and is_baseline")

@patch
def get_replication_baseline(self:GBEProvider):
    df = self.get_gbe_data()
    return df.query("(is_baseline and (not initial)")
df = dp.get_gbe_data(update = True)
df = dp.get_gbe_data()
df.reset_index().gbe_index.apply(lambda x: x.split('_')[1]).value_counts()<jupyter_output><empty_output><jupyter_text>Checking distance between sessions<jupyter_code>def test_time_since_last_session(df):
    fig, ax = plt.subplots(figsize=(15,6))
    df['is_baseline_check'] = df.session_number<3
    ax = sns.countplot(x = 'time_since_last_session', hue = 'is_baseline', data = df.query('(time_since_last_session < 60)'))
    for index, label in enumerate(ax.xaxis.get_ticklabels()):
        label.set_visible(False)
        if index%5==0:
            label.set_visible(True)
test_time_since_last_session(dp.get_gbe_data())
sns.despine()<jupyter_output><empty_output><jupyter_text>Checking compliance<jupyter_code>def test_compliance(_df):
    basedate = pd.to_datetime(_df.date.max())
    _df['study_days_since_start'] = (basedate - _df['starting_date']).dt.days
    _df.loc[_df['study_days_since_start'] >= 392, 'study_days_since_start'] = 392
    
    _df['expected_ema_sessions'] = (_df.study_days_since_start - 28)/28
    _df[['starting_date','study_days_since_start','expected_ema_sessions']]
    _df['gbe_compliance'] = _df.total_ema_sessions_completed / _df.expected_ema_sessions
    _df.loc[_df['gbe_compliance'] >= 1, 'gbe_compliance'] = 1
    _df.loc[_df['gbe_compliance'] <= 0, 'gbe_compliance'] = 0
    ax = _df.groupby('participant').first().gbe_compliance.hist(bins=20)
    ax.grid(False)
    sns.despine()
    print(_df.groupby('participant').first().gbe_compliance.describe())
test_compliance(dp.get_gbe_data())<jupyter_output>count    477.000000
mean       0.675454
std        0.357345
min        0.000000
25%        0.384615
50%        0.843373
75%        1.000000
max        1.000000
Name: gbe_compliance, dtype: float64<jupyter_text>Decoding GBE stringsGBE data gets stored in strings, which will have to be decoded before they are useable for later analyses:<jupyter_code>df.FruitTapGame.iloc[0]
#export 
@patch
def decode_gbe_string(self:GBEProvider, s):
    """This function function turns one gbe output string into a dataframe"""
    def replace(g):
        return g.group(0).replace(',', '|')
    s = re.sub(r'\[.*?\]', replace, s) # The comma separated string can contain comma separated list items
    columns, df = s.replace('","',';').replace('"','').split('\n')
    df = pd.DataFrame([column.split(',') for column in df.split(';')][:-1]).transpose().ffill().iloc[:-1]
    df.columns = [c.replace('tr_','') for c in columns.split(',')[:-1]]
    def to_datetime(x):
        return pd.to_datetime(x[1:-1])
    if 'timestarted' in df.columns:
        df['timestarted'] = df.timestarted.apply(to_datetime)
        df['timesubmitted'] = df.timesubmitted.apply(to_datetime)
        df.drop(columns = 'appversion', inplace = True)
    else:
        df['timestarted'] = pd.to_datetime(np.nan)
        df['timesubmitted'] = pd.to_datetime(np.nan)
    return df<jupyter_output><empty_output><jupyter_text>Decoded data examplesHere are some examples of some decoded data strings.  We will further process this data in the specific task notebooks.<jupyter_code>rh_times = df['RewardAndHappinessGame'].apply(get_time_started)

def get_time_started(s):
    try:
        df = dp.decode_gbe_string(s)
        time_started = df.timestarted.iloc[0]
    except:
        time_started = np.nan
    return time_started


ft_times = df['FruitTapGame'].apply(get_time_started)
wm_times = df['WorkingMemoryGame'].apply(get_time_started)
cg_times = df['CardGame'].apply(get_time_started)
rh_times = df['RewardAndHappinessGame'].apply(get_time_started)
test = pd.DataFrame({
'ft':ft_times,
'wm':wm_times,
    'cg':cg_times,
    'rh':rh_times
})
#test = (test['ft'] - test['wm']).dt.total_seconds()
test.isna().sum()<jupyter_output><empty_output><jupyter_text>SSRT<jupyter_code>dp.decode_gbe_string(df.FruitTapGame.iloc[0])<jupyter_output><empty_output><jupyter_text>Working memory game<jupyter_code>dp.decode_gbe_string(df.WorkingMemoryGame.iloc[0])<jupyter_output><empty_output><jupyter_text>Card game<jupyter_code>dp.decode_gbe_string(df.CardGame.iloc[0])<jupyter_output><empty_output><jupyter_text>Gambling task<jupyter_code>dp.decode_gbe_string(df.RewardAndHappinessGame.iloc[0])<jupyter_output><empty_output><jupyter_text>Decoding all sessions at once<jupyter_code>#export
@patch
def decode_gbe_strings(self:GBEProvider, df, column):
    '''This function turns all gbe output strings in a column into dataframes and concatenates them.'''
    df = df[~df[column].isna()] # Selecting non nan data
    #gbe_data = pd.concat(df.set_index(['participant','session_number'])[column].apply(self.decode_gbe_string).values, keys = df.index)
    #gbe_data = pd.concat(df[column].apply(self.decode_gbe_string).values)
    df = pd.concat(df[column].apply(self.decode_gbe_string).values, keys = df.index)
    df.index.rename('trial_number',level = 1, inplace = True)
    df = df.reset_index()
    df['trial_number'] = df.trial_number + 1
    return df<jupyter_output><empty_output><jupyter_text>We double check that the decoding worked by testing it against our earlier defined `find_gbe_game_starting_time` function.<jupyter_code>def test_decoding(df, decoded):
    df = df[~df.FruitTapGame.isna()]
    decoded = decoded.groupby('gbe_index').first()
    for loc in range(len(decoded)):
        time_started_1 = dp.find_gbe_game_starting_time(df.iloc[loc].FruitTapGame)
        time_started_2 = decoded.iloc[loc]['timestarted']
        both_nan = (pd.isnull(time_started_1)) and (pd.isnull(time_started_1))
        equal = time_started_1 == time_started_2
        assert equal or both_nan, "Decoding problem at %d %s != %s"%(loc, str(time_started_1), str(time_started_2))
    print("Check passed.")
    
test_decoding(df, decoded)<jupyter_output>Check passed.
{'fraction_non_alphanumeric': 0.12024641630138609, 'fraction_numerical': 0.011965406942305414, 'mean_word_length': 5.262240356083086, 'pattern_counts': {'":': 1, '<': 83, '<?xml version=': 0, '>': 86, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '251940', 'license': 'Apache-2.0', 'n_tokens_mistral': 5893, 'n_tokens_neox': 5615, 'n_words': 1224, 'path': 'notebooks/02_gbe.data_provider.ipynb', 'repo_name': 'hgzech/trr265', 'size': '16882'}