<jupyter_start><jupyter_text>Activity Recognition using Cell Phone Accelerometers by <NAME>, <NAME>, <NAME> Data mining<jupyter_code># Loading all the needed modules
import pandas as pd
import matplotlib.pyplot as plt
import matplotlib
import seaborn as sns
import numpy as np
import scipy as sp
from mpl_toolkits.mplot3d import Axes3D
%matplotlib inline
matplotlib.rcParams['figure.figsize'] = (8, 6)<jupyter_output><empty_output><jupyter_text>NotesThere is a falsy line in the original file. The following bash code fixes that:```cat WISDM_ar_v1.1_raw.txt | awk -F',' 'BEGIN { OFS=","; RS=";"; ORS=";"; }{ if (NF==6) { print $0,"" } else if (NF==7) { print $0 }}' > raw_data_fixed.txt```<jupyter_code>root = '/Users/samiadrappeau/data_science/projets_perso/liberty_rider/'
dirin = 'WISDM/data_WISDM/'
#fid = root + dirin + 'raw_data_fixed.txt'
fid = root + dirin + 'WISDM_ar_v1.1_raw.txt'
# Loading data
df = pd.read_csv(fid, lineterminator=';' ,
    header=None ,names=['user','activity','timestamp','xaccel','yaccel','zaccel', 'NA'])
del df ['NA']
print('Shape of data frame is:', df.shape, sep=" ")
df[['xaccel', 'yaccel', 'zaccel']].describe()
df.head()
def create_segments(df):
    ''' Convert timestamp to second, sort by user and time and create 10-second segments
    '''
    # Converting timestamp in nano-second to second
    df['time_seconds'] = df['timestamp']*1e-9
    # Sorting data by user and timestamp
    df = df.sort_values(by=['user', 'time_seconds'])
    # create segments
    segment_length = 200
    df['seq'] = range(df.shape[0])
    df['segment'] = df.groupby(['user','activity'])['seq'].apply(lambda x: x%segment_length == 0).fillna(0).cumsum()
    return df
df_segmented = create_segments(df)
df_segmented.head()
#fig = plt.figure()
#ax1 = fig.add_subplot(111)
#plt.plot(df_segmented.loc[(df_segmented['user'] == 2) & (df_segmented['activity'] == 'Walking')]['timestamp'].diff(), '*')
#ax1.set_ylim([-0.5*1e9,0.2*1e9])
print('Shape of data frame is now:', df_segmented.shape, sep=" ")
#df_segmented.head(201)
# Extracting a user activity
df_user = df_segmented[df_segmented.user == 33]
print(df_user.head())
print(df_user.shape)
df_user_activity = df_user[df_user.activity == 'Sitting']
print(df_user_activity.head())
print(df_user_activity.shape)
df_user_activity[df_user_activity.segment==4651].plot(x='time_seconds', y=['xaccel',  'yaccel',  'zaccel'], subplots=True, layout=[3,1])
print(df_user_activity['segment'].unique())
# FIXME remove this for loop with a lambda function
for x in df_user_activity.segment.unique():
    print(df_user_activity[df_user_activity.segment == x][['xaccel','yaccel', 'zaccel']].describe())
print(len(df_segmented['segment'].unique()),len(df_segmented['user'].unique()))
range((df_segmented['segment'].unique()).shape[0])
print(df_segmented.columns)
def create_new_features(df_old):
    # Sorting data by user and segment
    accel_cols = ['xaccel','yaccel','zaccel']
    df_new = df_old.loc[:,accel_cols+['user','segment','activity']].groupby(['user', 'segment','activity'])
    
    # acceleration means
    means = df_new[accel_cols].apply(lambda x: np.mean(x))
    
    # acceleration standard deviation
    sds = df_new[accel_cols].apply(lambda x: np.std(x))
    
    # avg absolute difference
    avg_absolute_diff_1 = df_new[accel_cols[0]].apply(lambda x: np.mean(np.absolute(x-np.mean(x))))
    avg_absolute_diff_2 = df_new[accel_cols[1]].apply(lambda x: np.mean(np.absolute(x-np.mean(x))))
    avg_absolute_diff_3 = df_new[accel_cols[2]].apply(lambda x: np.mean(np.absolute(x-np.mean(x))))
    avg_absolute_diffs = [ avg_absolute_diff_1, avg_absolute_diff_2, avg_absolute_diff_3 ]
    
    # avg resultant accel
    # FIXME problem here because x are a collection of Series.
    avg_resultant_accel = df_new[accel_cols].apply(lambda x: np.mean(np.sqrt(x.iloc[:,0]**2+
                                                                        x.iloc[:,1]**2+
                                                                        x.iloc[:,2]**2)))
    # time between peaks
    between_peaks_1 = df_new[accel_cols[0]].apply(lambda x: np.abs(np.argmax(x)-np.argmin(x)))
    between_peaks_2 = df_new[accel_cols[1]].apply(lambda x: np.abs(np.argmax(x)-np.argmin(x)))
    between_peaks_3 = df_new[accel_cols[2]].apply(lambda x: np.abs(np.argmax(x)-np.argmin(x)))
    between_peaks = [ between_peaks_1, between_peaks_2, between_peaks_3 ]
    
    
    # concat columns
    X = pd.concat([means, sds] + avg_absolute_diffs + between_peaks + [avg_resultant_accel], axis=1)
    
    X.columns = ['xavg', 'yavg', 'zavg','xstanddev','ystanddev', 'zstanddev','xabsdiff', 'yabsdiff', 'zabsdiff', 'xpeak', 'ypeak', 'zpeak', 'resultant']
    X = X.reset_index(level=['user', 'segment', 'activity'])
    X.drop('segment', axis=1, inplace=True)
    X = X[['user',\
           'xavg', 'yavg', 'zavg',\
           'xpeak', 'ypeak','zpeak',\
           'xabsdiff', 'yabsdiff', 'zabsdiff',\
           'xstanddev', 'ystanddev', 'zstanddev',\
           'resultant',\
           'activity']]
    return X
df_new_feats = create_new_features(df_segmented)
df_new_feats[(df_new_feats.user==33) & (df_new_feats.activity=='Jogging')].head()
df_new_feats.columns
df_new_feats.columns
df_new_feats.head()
df_new_feats[df_new_feats.user == 1].head()
df_new_feats.columns<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16704119850187266, 'fraction_numerical': 0.01441947565543071, 'mean_word_length': 4.51186790505676, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '160857', 'license': 'MIT', 'n_tokens_mistral': 2017, 'n_tokens_neox': 1962, 'n_words': 355, 'path': 'data_mining_wisdm.ipynb', 'repo_name': 'SamAstro/WIDSM', 'size': '5369'}