<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
from joblib import load

train = load("/content/drive/MyDrive/NCC Train Test Set (Resampled 20Hz)/Split 80 20/TRAIN.joblib")
test = load("/content/drive/MyDrive/NCC Train Test Set (Resampled 20Hz)/Split 80 20/TEST.joblib")

print('Train set shape: ', train.shape)
print('Test set shape: ', test.shape)<jupyter_output>Train set shape:  (4875587, 5)
Test set shape:  (1218899, 5)<jupyter_text>Sliding Window<jupyter_code>def sliding_window(df, num_datapoint = 50, slide_frac = 0.5):
  import pandas as pd
  import numpy as np
  from tqdm.notebook import tqdm
  acc_x = pd.DataFrame()
  acc_y = pd.DataFrame()
  acc_z = pd.DataFrame()
  label = pd.DataFrame()
  slide = (int)(num_datapoint * slide_frac)
  take_index = df['take'].unique()
  
  for i in tqdm(take_index):
    init = 0
    end = num_datapoint
    temp_df = df.loc[df['take'] == i]
    while end <= temp_df.shape[0]:
      acc_x = acc_x.append(temp_df['x'].iloc[init:end].reset_index(drop=True).transpose(),ignore_index=True)
      acc_y = acc_y.append(temp_df['y'].iloc[init:end].reset_index(drop=True).transpose(),ignore_index=True)
      acc_z = acc_z.append(temp_df['z'].iloc[init:end].reset_index(drop=True).transpose(),ignore_index=True)
      label = label.append(temp_df['label'].iloc[init:end].reset_index(drop=True).transpose(),ignore_index=True)
      init = init + slide
      end = end + slide
  label = label.median(axis=1)    
  
  print('acc_x = \n')
  display(acc_x)
  print('\n\n---------------\n\nacc_y = \n')
  display(acc_y)
  print('\n\n---------------\n\nacc_z = \n')
  display(acc_z)
  print('\n\n---------------\n\nlabel = \n')
  display(label)

  print('\n\nDone')

  return acc_x, acc_y, acc_z, label
#----------------------------- Window Train Set ---------------------------------

acc_x_train, acc_y_train, acc_z_train, label_train = sliding_window(train, num_datapoint = 1000, slide_frac = 0.05) 
#----------------------------- Export Train joblib ---------------------------------

from joblib import dump

export_path = '/content/drive/MyDrive/NCC Train Test Set (Resampled 20Hz)/Split 80 20/Windowed 1000/'

print('Exporting joblib at: ', export_path + 'acc_x_train.joblib', '...', end='')
dump(acc_x_train, export_path + 'acc_x_train.joblib')
print('Done')
print('Exporting joblib at: ', export_path + 'acc_y_train.joblib', '...', end='')
dump(acc_y_train, export_path + 'acc_y_train.joblib')
print('Done')
print('Exporting joblib at: ', export_path + 'acc_z_train.joblib', '...', end='')
dump(acc_z_train, export_path + 'acc_z_train.joblib')
print('Done')
print('Exporting joblib at: ', export_path + 'label_train.joblib', '...', end='')
dump(label_train, export_path + 'label_train.joblib')
print('Done')
#----------------------------- Window Test Set ---------------------------------

acc_x_test, acc_y_test, acc_z_test, label_test = sliding_window(test, num_datapoint = 1000, slide_frac = 0.05) 
#----------------------------- Export Test joblib ---------------------------------

from joblib import dump

export_path = '/content/drive/MyDrive/NCC Train Test Set (Resampled 20Hz)/Split 80 20/Windowed 1000/'

print('Exporting joblib at: ', export_path + 'acc_x_test.joblib', '...', end='')
dump(acc_x_test, export_path + 'acc_x_test.joblib')
print('Done')
print('Exporting joblib at: ', export_path + 'acc_y_test.joblib', '...', end='')
dump(acc_y_test, export_path + 'acc_y_test.joblib')
print('Done')
print('Exporting joblib at: ', export_path + 'acc_z_test.joblib', '...', end='')
dump(acc_z_test, export_path + 'acc_z_test.joblib')
print('Done')
print('Exporting joblib at: ', export_path + 'label_test.joblib', '...', end='')
dump(label_test, export_path + 'label_test.joblib')
print('Done')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.2231045081967213, 'fraction_numerical': 0.01767418032786885, 'mean_word_length': 3.127906976744186, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '392588', 'license': 'Apache-2.0', 'n_tokens_mistral': 1450, 'n_tokens_neox': 1374, 'n_words': 261, 'path': 'SlidingWindow_Nurse_Care_Dataset.ipynb', 'repo_name': 'farhanfuadabir/Nurse-Care-Challenge-2020', 'size': '3904'}