<jupyter_start><jupyter_text>This Notebook is meant for the analysis of demographics survey data and extract the appropriate set of subjects from the mPower data.<jupyter_code>import pandas as pd
import numpy as np
import seaborn as sns
from matplotlib import pyplot as plt
#from pydub import AudioSegment
import scipy.signal as sg
import sys
data = pd.read_csv('..\\..\\Data\\Demographics\\Demographics_Survey.csv');
data = data.drop(columns = ['ROW_ID','ROW_VERSION','createdOn','appVersion','phoneInfo']);
#the feature that points to whether a person has Parkinson's or not is the professional diagnosis column. 
#since, smoking can effect the voice samples, I am excluding them from the dataset for the true distinction between Parkinson's 
#and healthy patients.
#age is set to greater than 42 because clinically diagnosed samples are concentrated after 42.
#lets ignore health history for now! 
plt.figure(figsize=(10,5))
sns.distplot(data[data['professional-diagnosis'] == True].dropna().age,label='Parkinson age distribution');
sns.distplot(data[data['professional-diagnosis'] == False].dropna().age,label='Healthy age distribution',bins=13);
plt.legend();
demo_data = data.loc[(data['smoked'] == False)]; #(data['age'] > 44)
demo_data = demo_data.dropna(subset=['professional-diagnosis'])
print('Parkinson\'s Patient count:',len(demo_data[demo_data['professional-diagnosis'] == True]))
print('Healthy Patient count:',len(demo_data[demo_data['professional-diagnosis'] == False]))
#data['medication-start-year'].unique()
#print(len(temp[temp['professional-diagnosis'] == False]))
demo_data.to_csv('rep_study_cohort.csv')
#these are the 1853 people who form my study cohort
#this pre-processing is useful but unnecessary at this stage
#we should do this after taking intersection in lists in each activity!
from sklearn.model_selection import StratifiedShuffleSplit
sk1 = StratifiedShuffleSplit(n_splits = 1, test_size=0.20, random_state = 42)
sk2 = StratifiedShuffleSplit(n_splits = 1, test_size=0.125, random_state = 42)
X = demo_data[['healthCode','age']].values
y = demo_data['professional-diagnosis'].values
for train_index, test_index in sk1.split(X, y):
    #print("TRAIN:", train_index, "TEST:", test_index)
    X_temp, X_test = X[train_index], X[test_index]
    y_temp, y_test = y[train_index], y[test_index]

for train_index, test_index in sk2.split(X_temp, y_temp):
    #print("TRAIN:", train_index, "TEST:", test_index)
    X_train, X_val = X_temp[train_index], X_temp[test_index]
    y_train, y_val = y_temp[train_index], y_temp[test_index]
    
train_split = pd.DataFrame(np.hstack((X_train,np.expand_dims(y_train,axis=1))),\
                           columns = ['healthCode','age','professiona-diagnosis'])
val_split = pd.DataFrame(np.hstack((X_val,np.expand_dims(y_val,axis=1))),\
                           columns = ['healthCode','age','professiona-diagnosis'])
test_split = pd.DataFrame(np.hstack((X_test,np.expand_dims(y_test,axis=1))),\
                           columns = ['healthCode','age','professiona-diagnosis'])<jupyter_output><empty_output><jupyter_text>From here onwards, voice activity also comes into play in this file!However, it contains very useful code on how to filter out those patients who actually performed the activity in each individual activity<jupyter_code>voice_data = pd.read_csv('..\\..\\Data\\VoiceData\\Voice Activity.csv')
voice_data.drop(columns = ['ROW_ID','ROW_VERSION','appVersion','phoneInfo','createdOn'],inplace=True)
voice_data = voice_data[voice_data['medTimepoint'] != 'Just after Parkinson medication (at your best)'];
voice_data[voice_data['healthCode'] == '67bdd316-26fc-4fc7-8431-bf9f41a649dd']
#voice_data.set_index('healthCode').loc['67bdd316-26fc-4fc7-8431-bf9f41a649dd']
#this block takes the intersection between people selected in demographics data and the people whose recordings we are using
c1 = voice_data['healthCode'].unique().tolist();
c2 = demo_data['healthCode'].unique().tolist();
c3 = list(filter(lambda x: x in c1, c2))
print(len(c2))
voice_data = voice_data.set_index(['healthCode','recordId']).sort_index();
voice_data.head()
voice_data = voice_data.loc[c3,:]
voice_data.head()
#now, only these are the audio files that are necessary!
#constructing target series with respect to healthCode
Y = demo_data.set_index('healthCode').loc[c3]['professional-diagnosis']
#len(voice_data)
#so, so far Y contains the target variables with respoect to healthCode
#and voice_data contains all the healthCode audio_file codes and recordIds (which contain the repitition of activity by
#the same person)
voice_data.index = voice_data.index.droplevel(1)
print('Total',len(voice_data),'patients');
voice_data.head()
#this command deletes the record Ids and awards duplicate healthcode indices to multiple recordings by the same patient
Ytemp = Y[voice_data.index.values];
nullind = np.where(np.asanyarray(np.isnan(Ytemp.values.astype(float))))[0].tolist()
voice_data = voice_data.reset_index().drop(nullind).set_index('healthCode');
#this block drops the values where we have nan as parkinson professional diagnosis value
def update_progress(progress):
    barLength = 30 # Modify this to change the length of the progress bar
    status = ""
    if isinstance(progress, int):
        progress = float(progress)
    if not isinstance(progress, float):
        progress = 0
        status = "error: progress var must be float\r\n"
    if progress < 0:
        progress = 0
        status = "Halt...\r\n"
    if progress >= 1:
        progress = 1
        status = "Done...\r\n"
    block = int(round(barLength*progress))
    text = "\rPercent: [{0}] {1}% {2}".format( "#"*block + "-"*(barLength-block), progress*100, status)
    sys.stdout.write(text)
    sys.stdout.flush()
pad = lambda a,i : a[0:i] if len(a) > i else np.concatenate((a, [0] * (i-len(a)))) #for fixing things to a fixed length
sample_len = 44100; #sample Fs is 44100. After 10x decimation, a 10-sec recording should have 44.1k Samples
err_samples = [];
j = True;
k = 0;
parts = 500;
#X = np.array((sample_len));
#lets start extracting samples!

for i in range(len(voice_data)):
    path = 'D:\\Projects\\Parkinson Diagnosis\\mPower\\Voice Data\\Audio\\'+str(voice_data['audio_audio.m4a'].iloc[i]) + '.m4a';
    try:
        audio = np.array(AudioSegment.from_file(path).get_array_of_samples().tolist());
        audio = audio/audio.max() #Normalize the PCM waveform
        deci = sg.decimate(x=audio,q=10) #lets decimate
        deci = pad(deci,sample_len); #lets fix the sample length! so that we have no problems in concatenation!
        #plt.plot(np.abs(np.fft.fftshift(np.fft.fft(deci)))) #If you want to visualize the fft of the thing!
        if(j is True): 
            X = deci;
            X = np.expand_dims(X,axis=0);
            j = False;
        else: X = np.vstack((X,deci))
        
        if(X.shape[0] % parts == 0):
            np.save('X_sec'+str(k)+'.npy',X)
            k = k + 1;
            j = True;
    except:
        err_samples.append(i);
    update_progress((i+1)/len(voice_data));
np.save('X_sec'+str(k)+'.npy',X)
voice_data = voice_data.reset_index().drop(err_samples).set_index('healthCode');
voice_data.head()
len(voice_data)
Y = Y[voice_data.index.values];
##Now, Y is a series with indications of which recording is Parkinson's or not
#(Y==True).sum()/(Y == False).sum()
print(Y.shape)
Y.head()
#Y.index[Y.apply(np.isnan)]

#Y.values.astype(float)
#35347 -((Y==True).sum() + (Y == False).sum())
#np.save('Y.npy',Y)
(Y==False).sum()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14158822749569136, 'fraction_numerical': 0.017632241813602016, 'mean_word_length': 4.754385964912281, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '175376', 'license': 'MIT', 'n_tokens_mistral': 2580, 'n_tokens_neox': 2429, 'n_words': 634, 'path': 'Scripts/Demographics/mPowerEDA.ipynb', 'repo_name': 'khizar-anjum/mPowerAnalysis', 'size': '7543'}