<jupyter_start><jupyter_text>Plotting age distributions with respect to genotype groups<jupyter_code>%matplotlib inline
import pandas as pd
from scipy import stats
from matplotlib import pyplot as plt

data = pd.read_excel('/home/grg/spm/data/covariates.xls')

for i in xrange(5):
    x = data[data['apo'] == i]['age'].values
    plt.hist(x, bins=20)
    print i, 'W:%.4f p:%.4f -'%stats.shapiro(x), len(x), 'subjects between', int(min(x)), 'and', int(max(x))

plt.legend(['apoe23', 'apoe24', 'apoe33', 'apoe34', 'apoe44'])
plt.show()<jupyter_output>0 W:0.9705 p:0.0222 - 102 subjects between 46 and 75
1 W:0.9539 p:0.0769 - 44 subjects between 45 and 71
2 W:0.9613 p:0.0005 - 143 subjects between 45 and 75
3 W:0.9640 p:0.0004 - 160 subjects between 45 and 74
4 W:0.9306 p:0.0013 - 65 subjects between 45 and 67<jupyter_text>For two of the 5 groups, the Shapiro test p-value is lower than 1e-3, which means that the distributions of these two groups can't be considered as normal. (But theorically none of them is)  Matching pairs using nearest neighboursThe matching algorithm:<jupyter_code>from sklearn.preprocessing import StandardScaler
from sklearn.neighbors import NearestNeighbors

def get_matching_pairs(treated_df, non_treated_df, scaler=True):

    treated_x = treated_df.values
    non_treated_x = non_treated_df.values
    if scaler:
        scaler = StandardScaler()
        scaler.fit(treated_x)
        treated_x = scaler.transform(treated_x)
        non_treated_x = scaler.transform(non_treated_x)

    nbrs = NearestNeighbors(n_neighbors=1, algorithm='ball_tree').fit(non_treated_x)
    distances, indices = nbrs.kneighbors(treated_x)
    indices = indices.reshape(indices.shape[0])    
    matched = non_treated_df.ix[indices]
    matched = non_treated_df.irow(matched.index)
    return matched<jupyter_output><empty_output><jupyter_text>Loading data<jupyter_code>df = pd.read_excel('/home/grg/spm/data/covariates.xls')
df = df[['subject','apo','age','gender','educyears']]

groups = [df[df['apo']==i] for i in xrange(5)]
for i in xrange(5):
    groups[i] = groups[i].set_index(groups[i]['subject'])
    del groups[i]['subject']
    del groups[i]['apo']<jupyter_output><empty_output><jupyter_text>Matching the groups<jupyter_code>treated_df =  groups[4]
matched_df = [get_matching_pairs(treated_df, groups[i], scaler=False) for i in xrange(4)]<jupyter_output>/home/grg/jupyter/lib/python2.7/site-packages/ipykernel/__main__.py:18: FutureWarning: irow(i) is deprecated. Please use .iloc[i]<jupyter_text>Plotting data and see that the groups are now matching<jupyter_code>fig, ax = plt.subplots(figsize=(6,6))

for i in xrange(4):
    x = matched_df[i]['age']
    plt.hist(x, bins=20)   
    print i, 'W:%.4f p:%.4f -'%stats.shapiro(x), len(x), 'subjects between', int(min(x)), 'and', int(max(x))
    
x = treated_df['age']
plt.hist(x, bins=20)   
print 4, 'W:%.4f p:%.4f -'%stats.shapiro(x), len(x), 'subjects between', int(min(x)), 'and', int(max(x))

plt.legend(['apoe23', 'apoe24', 'apoe33', 'apoe34', 'apoe44'])<jupyter_output>0 W:0.9165 p:0.0003 - 65 subjects between 46 and 66
1 W:0.9271 p:0.0009 - 65 subjects between 45 and 67
2 W:0.9295 p:0.0012 - 65 subjects between 45 and 68
3 W:0.9163 p:0.0003 - 65 subjects between 46 and 68
4 W:0.9306 p:0.0013 - 65 subjects between 45 and 67<jupyter_text>Matching groups using linear assignment method<jupyter_code>import pandas as pd
df = pd.read_excel('/home/grg/spm/data/covariates.xls')
df = df[['subject','apo','age','gender','educyears']]

groups = [df[df['apo']==i] for i in xrange(5)]
for i in xrange(5):
    groups[i] = groups[i].set_index(groups[i]['subject'])
    del groups[i]['subject']
    del groups[i]['apo']
groups = [df[df['apo']==i] for i in xrange(5)]
for i in xrange(5):
    groups[i] = groups[i].set_index(groups[i]['subject'])
    del groups[i]['apo']
    del groups[i]['subject']
treated_df =  groups[4]
non_treated_df = groups[0]
from scipy.spatial.distance import cdist
from scipy import optimize

def get_matching_pairs(treated_df, non_treated_df):
    cost_matrix = cdist(treated_df.values, non_treated_df.values)
    row_ind, col_ind = optimize.linear_sum_assignment(cost_matrix)
    return non_treated_df.iloc[col_ind]
treated_df =  groups[4]
matched_df = [get_matching_pairs(treated_df, groups[i]) for i in xrange(4)]<jupyter_output><empty_output><jupyter_text>Plotting data and see that the groups are now matching<jupyter_code>fig, ax = plt.subplots(figsize=(6,6))

for i in xrange(4):
    x = matched_df[i]['age']
    plt.hist(x, bins=20)   
    print i, 'W:%.4f p:%.4f -'%stats.shapiro(x), len(x), 'subjects between', int(min(x)), 'and', int(max(x))
    
x = treated_df['age']
plt.hist(x, bins=20)   
print 4, 'W:%.4f p:%.4f -'%stats.shapiro(x), len(x), 'subjects between', int(min(x)), 'and', int(max(x))

plt.legend(['apoe23', 'apoe24', 'apoe33', 'apoe34', 'apoe44'])
import json
groups_index = [each.index.tolist() for each in matched_df]
groups_index.append(groups[4].index.tolist())
json.dump(groups_index, open('/tmp/groups.json','w'))<jupyter_output><empty_output><jupyter_text>Assessing the effect from the matchingWe perform a two-sample t-test between each group and the target group, before and after applying the matching.As the dataset is composed of 3 variables (age, gender, education), this returns 3 t values and 3 p-values for each comparison.<jupyter_code>from scipy.stats import ttest_ind
for i in xrange(4):
    print '=== Group %s ==='%i
    tval_bef, pval_bef = ttest_ind(groups[i].values, treated_df.values)
    tval_aft, pval_aft = ttest_ind(matched_df[i].values, treated_df.values)
    print 'p-values before matching: %s - p-values after matching: %s'%(pval_bef, pval_aft)
df = pd.read_excel('/home/grg/spm/data/covariates.xls')
list(df[df['apo']!=1]['subject'].values)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15116678105696638, 'fraction_numerical': 0.044440631434454356, 'mean_word_length': 5.247588424437299, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '126653', 'license': 'MIT', 'n_tokens_mistral': 2407, 'n_tokens_neox': 2189, 'n_words': 517, 'path': 'notebooks/Ages distributions.ipynb', 'repo_name': 'xgrg/alfa', 'size': '5828'}