<jupyter_start><jupyter_text>Positioning project<jupyter_code># import libraries
import pandas as pd
import numpy as np
import scipy.fftpack
from scipy.signal import butter, lfilter, freqz
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
from sklearn.model_selection import train_test_split, cross_val_score
%matplotlib inline

def butter_lowpass(cutoff, fs, order=5):
    nyq = 0.5 * fs
    normal_cutoff = cutoff / nyq
    b, a = butter(order, normal_cutoff, btype='low', analog=False)
    return b, a

def butter_lowpass_filter(data, cutoff, fs, order=5):
    b, a = butter_lowpass(cutoff, fs, order=order)
    y = lfilter(b, a, data)
    return y

def butter_bandpass(lowcut, highcut, fs, order=5):
    nyq = 0.5 * fs
    low = lowcut / nyq
    high = highcut / nyq
    b, a = butter(order, [low, high], btype='band')
    return b, a

def butter_bandpass_filter(data, lowcut, highcut, fs, order=5):
    b, a = butter_bandpass(lowcut, highcut, fs, order=order)
    y = lfilter(b, a, data)
    return y
def statfeatureextract(data_sliced, num_sample):
    order = 5
    cutoff = 20
    fs = num_sample/10
    lowcut = 0.1
    highcut = 20
    if flag_lowpass == 1:
        x_filtered = butter_lowpass_filter(data_sliced.x, cutoff, fs, order)
        y_filtered = butter_lowpass_filter(data_sliced.y, cutoff, fs, order)
        z_filtered = butter_lowpass_filter(data_sliced.z, cutoff, fs, order)
    else:
        x_filtered = butter_bandpass_filter(data_sliced.x, lowcut, highcut, fs, order)
        y_filtered = butter_bandpass_filter(data_sliced.y, lowcut, highcut, fs, order)
        z_filtered = butter_bandpass_filter(data_sliced.z, lowcut, highcut, fs, order)
    x_mean = x_filtered.mean()
    x_var = x_filtered.var()
    y_mean = y_filtered.mean()
    y_var = y_filtered.var()
    z_mean = z_filtered.mean()
    z_var = z_filtered.var()
    return {"num_sample": int(data_sliced.shape[0]),"x_mean":x_mean, "x_var":x_var , "y_mean": y_mean, 
            "y_var": y_var, "z_mean": z_mean, "z_var": z_var}

def createstatfeature(data, activity):
#   remove outliner
    data.loc[data.x > 30, "x"] = 30
    data.loc[data.y > 30, "y"] = 30
    data.loc[data.z > 30, "z"] = 30
    data.loc[data.x < -30, "x"] = -30
    data.loc[data.y < -30, "y"] = -30
    data.loc[data.z < -30, "z"] = -30
    
    time_diff = 10000
    samplesper10sec = []
    statfeature = pd.DataFrame([])
    count = 0
    # if time index is a constant, we'll assume it samples at 250 Hz
    if data.time[0] == data.time[1]:
        data.time = np.arange(data.time[0], data.time[0] + 4*len(data.time) ,4)
    # slice data into 10 second period and extract statistical features.
    while len(data.time) > 1500:
        num_sample = np.sum(np.less_equal(data.loc[:, 'time'], 
                                          data.time[0]+time_diff))
        samplesper10sec.append(num_sample)
        data_sliced = data.loc[range(num_sample),:]
        statfeature = statfeature.append(statfeatureextract(data_sliced, num_sample), ignore_index=True)
        data.drop(range(num_sample), inplace = True)
        data.reset_index(drop=True, inplace = True)
        
    # add label to extracted features dataframe. 
    labeldict = {'laying_down':1, 'sitting':2, 'standing':3, 'walking':4}
    num_period = len(statfeature.x_mean)
    labels = pd.DataFrame({'activity_name': [activity]*num_period, 
                           'activity_label': [labeldict[activity]]*num_period})
    # # statfeature.
    result = pd.concat([statfeature, labels], axis = 1)
    return result
headername = ["time", "x", "y", "z", "ind", "label"]
flag_lowpass = 1
data = pd.read_csv('data new/train/layingdown/1_android.sensor.accelerometer.data.csv',names = headername)
df1 = createstatfeature(data, "laying_down")
data = pd.read_csv('data new/train/sitting/1_android.sensor.accelerometer.data.csv',names = headername)
df2 = createstatfeature(data, "sitting")
data = pd.read_csv('data new/train/standing/1_android.sensor.accelerometer.data.csv',names = headername)
df3 = createstatfeature(data, "standing")
data = pd.read_csv('data new/train/walking/1_android.sensor.accelerometer.data.csv',names = headername)
df4 = createstatfeature(data, "walking")
data = pd.concat([df1, df2, df3, df4], ignore_index=True )
data.head()
x = data[['num_sample', 'x_mean', 'x_var', 'y_mean', 'y_var', 'z_mean', 'z_var']]
y = data[['activity_label']]
x_train, x_test, y_train, y_test = train_test_split(x, y, random_state = 0)

from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
lw = 1
colors = ['navy', 'turquoise', 'darkorange','red']
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
lda = LinearDiscriminantAnalysis(n_components=3)
x_lda = lda.fit_transform(x,y)
for color, i, target_name in zip(colors, [1, 2, 3, 4], data.activity_name.unique()):
    ax.scatter(x_lda[y.activity_label == i, 0], x_lda[y.activity_label == i, 1], 
                x_lda[y.activity_label == i, 2], color=color, alpha=.8, lw=lw, label=target_name)
plt.legend(loc='best', shadow=False, scatterpoints=1)
plt.title('LDA with 3 components')
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')

for color, i, target_name in zip(colors, [1, 2, 3, 4], data.activity_name.unique()):
    xs = x.loc[y.activity_label == i, 'x_var']
    ys = x.loc[y.activity_label == i, 'y_var']
    zs = x.loc[y.activity_label == i, 'z_var']
    ax.scatter(xs, ys, zs, s = 5, label=target_name)
    
ax.set_xlabel('x_var');ax.set_ylabel('y_var');ax.set_zlabel('z_var')
ax.set_xbound(0,20);ax.set_ybound(0,20);ax.set_zbound(0,20)
plt.legend(loc='best', shadow=False, scatterpoints=1)
plt.title("Statistic feature (var) in 3D ")
from sklearn.neighbors import KNeighborsClassifier

knn_clf = KNeighborsClassifier(n_neighbors = 5)
knn_clf.fit(x_train, y_train)
knn_clf.score(x_test, y_test)
accu = np.mean(cross_val_score(knn_clf, x, y.activity_label, cv=10))
print("average accuracy for knn(k=5) is {0}".format(round(accu,5)))
crosstab = pd.crosstab(y_test.activity_label, knn_clf.predict(x_test),
                          rownames=['True'], colnames=['Predicted'],
                          margins=True)
crosstab
## decision tree classifier
from sklearn.tree import DecisionTreeClassifier
dt_clf = DecisionTreeClassifier(random_state=0)
dt_clf.fit(x_train, y_train.activity_label)
accu = np.mean(cross_val_score(dt_clf, x, y.activity_label, cv=10))
print("average accuracy for decisiontree is {0}".format(round(accu,5)))
crosstab = pd.crosstab(y_test.activity_label, dt_clf.predict(x_test),
                          rownames=['True'], colnames=['Predicted'],
                          margins=True)
crosstab
#Header 1M/N?MM
## yooo<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14145324597974984, 'fraction_numerical': 0.02054794520547945, 'mean_word_length': 3.9903417533432393, 'pattern_counts': {'":': 7, '<': 8, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '106998', 'license': 'MIT', 'n_tokens_mistral': 2624, 'n_tokens_neox': 2492, 'n_words': 515, 'path': 'sandbox_withfilter.ipynb', 'repo_name': 'kuantingC/positioning', 'size': '6716'}