<jupyter_start><jupyter_text>Human Activity Recognition from Wearable Inertial Sensor Networks Part 2<jupyter_code>from google.colab import drive
drive.mount('/content/drive')<jupyter_output>Mounted at /content/drive<jupyter_text>Table of Contents - [Import library and define visualization parameters](Import-library-and-define-visualization-parameters)- [Declaration of functions used](Declaration-of-functions-used)- [Import data](Import-data)- [Feature engineering](Feature-engineering)- [Save data in pickle format in google drive](Save-data-in-pickle-format-in-google-drive)  Import library and define visualization parameters<jupyter_code>from __future__ import print_function
from matplotlib import pyplot as plt
from matplotlib.ticker import FuncFormatter
from matplotlib import cm
%matplotlib inline
from math import pi
import numpy as np
import pandas as pd
from pandas import DataFrame
from typing import List
import seaborn as sns
from scipy import stats

from sklearn import preprocessing
from sklearn.preprocessing import LabelEncoder

import warnings 
warnings.filterwarnings('ignore')

# Display several outputs on a notebook output
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
# Define certain standard parameters at the outset
pd.options.display.float_format = '{: .4f}'. format
np.set_printoptions(precision=4)
plt.style.use ('ggplot')
# Files
path_to_file = '/content/drive/MyDrive/myproject/outputdata'<jupyter_output><empty_output><jupyter_text>Declaration of functions used<jupyter_code>def mad(x):
  mad = x.mad()
  return mad
def ran(x):
  range = x.max() - x.min()
  return range
# Define a function to compute acceleration spherical coordinates
def add_accelero_spherical_coordinates(df:DataFrame) -> DataFrame:
  """Generate normal acceleration, colalitude and longitude from imu data.

  Args:
    df (DataFrame): IMU data required!

  Returns:
    DataFrame: The IMU DataFrame with normal acceleration, colalitude and longitude.
  """

  df['acc_norm'] = np.sqrt((df.acc_rf_x)**2 + (df.acc_rf_y)**2 + (df.acc_rf_z)**2)
  df["colat_acc"] = np.arccos(df["acc_rf_z"]/df["acc_norm"])
  df["long_acc"] = np.arctan2(df["acc_rf_y"],df["acc_rf_x"])

  return df
# Define a function to compute gyrometer spherical coordinates
def add_gyro_spherical_coordinates(df:DataFrame) -> DataFrame:
  """Generate normal gyrometer, colalitude and longitude from imu data.

  Args:
    df (DataFrame): IMU data required!

  Returns:
    DataFrame: The IMU DataFrame with normal gyrometer, colalitude and longitude.
  """
  
  df['gyro_norm'] = np.sqrt((df.gyro_rf_x)**2 + (df.gyro_rf_y)**2 + (df.gyro_rf_z)**2)
  df["colat_gyro"] = np.arccos(df["gyro_rf_z"]/df["gyro_norm"])
  df["long_gyro"] = np.arctan2(df["gyro_rf_y"],df["gyro_rf_x"])

  return df
def add_group_index(df:DataFrame, windows:int=3) -> DataFrame:
  """Generate group index with specific windows for data aggregation.

  Args:
    df (DataFrame): raw data required!
    windows (int, default value is 5 s): The aggregation window.

  Returns:
    DataFrame: The raw DataFrame containing group index for aggregration.
  """

  sample_size = int((679073/11544)*windows)
  index = df.index.tolist()
  group_index = np.cumsum([(id % sample_size) == 0 for id in index]).tolist()
  df['group_index'] = group_index

  return df
def compute_features_stats(df:DataFrame, features_to_stats:List[str] = None) -> DataFrame:
  """Generates the aggregated dataframe with the different statistics on the given variables.

  Args:
    df (DataFrame): The raw DataFrame containing group index for aggregration.
    features_to_stats (List): List of variables whose statistics you want to display after aggregation.

  Returns:
    DataFrame: The aggregated dataframe with the different statistics on the input variables.
  """

  stats_agg = [np.min, np.max, np.mean, np.median, np.std, stats.skew, stats.kurtosis, stats.iqr, stats.median_absolute_deviation, mad, ran]
  dict_agg = {var:stats_agg for var in features_to_stats}
  df_agg = df.groupby(['act','group_index', 'filename']).agg(dict_agg)
  df_agg.columns = ["_".join(x) for x in df_agg.columns.ravel()]
  df_agg = df_agg.reset_index(['act', 'group_index', 'filename'])
  df_agg = df_agg.drop('group_index', axis=1)
  df_agg = df_agg.sort_values("filename")
  df_agg = df_agg.interpolate(method='linear', axis=0).ffill().bfill()

  return df_agg<jupyter_output><empty_output><jupyter_text>Import data<jupyter_code>!cp /content/drive/MyDrive/myproject/outputdata/df_1.pkl /tmp/df_1.pkl
df = pd.read_pickle('/tmp/df_1.pkl')
df<jupyter_output><empty_output><jupyter_text>Feature Engineering  Acceleros and Gyros Features Engineering<jupyter_code>df = add_accelero_spherical_coordinates(df)
df = add_gyro_spherical_coordinates(df)
df<jupyter_output><empty_output><jupyter_text>Aggregate Data by time windows<jupyter_code>df = add_group_index(df)
df
df = df.astype({"act": np.int64})
df_columns = df.columns.to_list()
features_to_stats = [columns for columns in df_columns if columns not in ('filename','activity','person','group_index','act')]
features_to_stats
df.dtypes
df_agg = compute_features_stats(df, features_to_stats)
df_agg<jupyter_output><empty_output><jupyter_text>Save data in pickle format in google drive<jupyter_code>df_agg.to_pickle("/tmp/df_2.pkl")
!cp /tmp/df_2.pkl /content/drive/MyDrive/myproject/outputdata<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11260349586016559, 'fraction_numerical': 0.006255749770009199, 'mean_word_length': 5.92484076433121, 'pattern_counts': {'":': 1, '<': 28, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '168106', 'license': 'MIT', 'n_tokens_mistral': 1862, 'n_tokens_neox': 1800, 'n_words': 446, 'path': 'project/HuGaDB_gait_analysis_2.ipynb', 'repo_name': 'Niangmohamed/Human-Activity-Recognition', 'size': '5435'}