<jupyter_start><jupyter_text>Load Packages<jupyter_code>import warnings
warnings.filterwarnings('ignore')

import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Pseudocodethis code below briefly explains how the whole process works---------------```pythondata = raw_data()assign activity labelprevRow = nullforeach row in data:  if row.ActivityLabel is null:     row.ActivityLabel = prevRow.ActivityLabel  prevRow = row  group according to its activitygroups = grouping(data)apply sliding windows into each group_ = []foreach g in groups:  sp = split(g, SECONDS_WINDOW)  if sp is not empty:      _ += sp  convert each group to vectorvactors = []for subg in _:  for item in subg:     vector[item.sensorID] += 1 if item.SensorValue == "ON"       normalise(vector)  vectors.append(vector)  return vectors```---------------<jupyter_code>fileName = "annotated.sample"
dataDir = '../Data/twor.2009/'
columns = ["Date", "Time", "SensorID", "SensorValue", "ActivityLabel", "Status"]
raw = pd.read_csv(dataDir+fileName, sep=" ", names=columns, header=None)
# data = raw
data = raw[(raw["ActivityLabel"].notna()) | (raw["SensorValue"] != "OFF")]
data['DateTime'] =  data['Date'] + " "+data['Time']
data['Timestamp'] =  pd.to_datetime(data['DateTime'], format='%Y-%m-%d %H:%M:%S.%f')


len(data)
def transform(d):
    g = []
    activity = None
    currGroup = []
    d["na"] = d[["ActivityLabel"]].isna()
    
    for i, row in d.iterrows():
        if not row["na"]:
            currGroup.append(row)
            activity = row["ActivityLabel"]
            if row['Status'] == "end":
                g.append(currGroup)
                currGroup = []
        else:
            row["ActivityLabel"] = activity
            row["Status"] = "working"
            currGroup.append(row)
            
    g.append(currGroup)
    return g

g = transform(data)
SECONDS_WINDOW = 15*60
def toVector(act, cols):
    if (len(act)==0):
        return []
    
    rows = []
    start = act[0]["Timestamp"].timestamp()
    activityLabel = act[0]["ActivityLabel"]
    actinwindow = {}
    for a in act:
        t = a["Timestamp"].timestamp()
        interval = ((t-start)//SECONDS_WINDOW)
        
        # consider only a second that when changes occur
        if interval in actinwindow:
            actinwindow[interval].append(a)
        else:
            actinwindow[interval] = [a]
    
    
    
    colIndex = {}
    for c in range(len(cols)):
        colIndex[cols[c]] = c
    
    vec = []
    for interval in actinwindow:
        v = np.zeros(len(cols))
        normalise = 0
        for a in actinwindow[interval]:
            if a["SensorValue"] in ["O", "ON", "OF", "ONF", "OPEN", "PRESENT"]:
                i = colIndex[a["SensorID"]]
                v[i] += 1

                normalise += 1
            elif a["SensorValue"] in ["OFFF", "OFF", "CLOSE", "ABSENT"]:
                continue
            else:
                # check isnumber https://stackoverflow.com/questions/354038/how-do-i-check-if-a-string-is-a-number-float
                value = a["SensorValue"].replace('F','',1)
                isNumberic = value.replace('.','',1).isdigit()
                if isNumberic:
                    value = float(value);
                    i = colIndex[a["SensorID"]]
                    v[i] += value
                    normalise += value
                else:
                    raise Exception('Unknown keyword: '+a["SensorValue"])
                    
            
        vec.append([value/normalise for value in v])
    
    rows = pd.DataFrame(vec, columns=cols)
    rows["ActivityName"] = activityLabel
    return rows

vColumns = np.insert(np.sort(raw.SensorID.unique()), 0, "ActivityName", axis=0);
vectors = pd.DataFrame(columns=vColumns)
for activity in g:
    v = toVector(activity, vColumns)
    if len(v) > 0:
        vectors = vectors.append(v, ignore_index=True)

vectors.head()
# TODO: optimise by using aggregate function

# GroupID = 1
# def aggActivity(d):
#     global GroupID
#     na = d.isna()
#     if not na["ActivityLabel"] and d["Status"] == "end":
#         GroupID += 1
#         return GroupID - 1
#     return GroupID

# data['GroupID'] = data.apply(aggActivity, axis=1)

# SECONDS_WINDOW = 10*60

# StartInterval = None
# def splitTimestamp(d):
#     global StartInterval
#     global SECONDS_WINDOW
#     na = d.isna()
#     if not na["ActivityLabel"] and d["Status"] == "begin":
#         StartInterval = d["Timestamp"].timestamp()

#     t = d["Timestamp"].timestamp()
#     interval = ((t-StartInterval)//SECONDS_WINDOW)
#     return interval

# data['Interval'] = data.apply(splitTimestamp, axis=1)

# print(data.columns)
# g = data.groupby(["GroupID", "Interval", "SensorID"])
vectors.to_csv(dataDir+fileName+".feat", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14938524590163935, 'fraction_numerical': 0.007172131147540984, 'mean_word_length': 2.5115107913669066, 'pattern_counts': {'":': 3, '<': 9, '<?xml version=': 0, '>': 10, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '25895', 'license': 'MIT', 'n_tokens_mistral': 1545, 'n_tokens_neox': 1501, 'n_words': 356, 'path': 'Reports/v0/Data Preprocessing.ipynb', 'repo_name': 'hillshadow/continual-learning-for-HAR', 'size': '4880'}