<jupyter_start><jupyter_text>Data Paths **ALWAYS RUN THE FOLLOWING CELL**  Here we specify the files where the algorithm takes the data and trained models from.<jupyter_code>experimentDir = 'model-nl-10-normalized'
modelDir = 'model'
trainingDataFile = 'training-data.txt'
testingDataFile = 'testing-data.txt'

runLoadData = True
runTrainModel = True
runTestModel = True

modelsPath = os.path.join('..', 'data', 'models')
dataPath = os.path.join('..', 'data', '08-07-19')
experimentPath = os.path.join(modelsPath, experimentDir)
modelPath = os.path.join(experimentPath, modelDir)
trainingDataPath = os.path.join(experimentPath, trainingDataFile)
testingDataPath = os.path.join(experimentPath, testingDataFile)

if not os.path.isdir(experimentPath):
    os.mkdir(experimentPath)<jupyter_output><empty_output><jupyter_text>Load measurement data<jupyter_code>def readBonsai(path):
    bonsai = pd.read_csv(path)
    bonsai = bonsai[['accX', 'accY', 'accZ', 'gyrX', 'gyrY', 'gyrZ']]
    return bonsai
    
def readEXLS3(path):
    exl = pd.read_fwf(path)
    exl.columns = exl.iloc[2]
    exl = exl[['a_x [g]:', 'a_y [g]:', 'a_z [g]:', 'ar_x [rad/s]:', 'ar_y [rad/s]:', 'ar_z [rad/s]:']]
    exl.rename(index=int, columns={
        'a_x [g]:': 'accX', 'a_y [g]:': 'accY', 'a_z [g]:': 'accZ', 
        'ar_x [rad/s]:': 'gyrX', 'ar_y [rad/s]:': 'gyrY', 'ar_z [rad/s]:': 'gyrZ'
    }, inplace=True)
    exl = exl.iloc[3:]
    exl.reset_index(drop=True, inplace=True)
    exl = exl.apply(pd.to_numeric)
    exl = exl.multiply(9.80665)
    return exl

def tagColumnNames(df, tag):
    newColumnNames = {columnName: columnName + tag for columnName in df.columns}
    return df.rename(index=int, columns=newColumnNames)


fileNameLocationMap = {
    'I-L9H': 'hip-r',
    'I-74V': 'hip-l',
    'I-WXB': 'knee-l',
    'I-0GN': 'knee-r',
    'I-2VZ': 'knee-r',
    'Gait - R': 'foot-r',
    'Gait - L': 'foot-l'
}
def mapFileNameToLocation(fileName):
    for name, location in fileNameLocationMap.items():
        if (name in fileName):
            return location
    return 'unknown'

def loadMeasurements(path):
    measurements = {}
    for fileOrDir in os.listdir(path):
        if (fileOrDir.endswith('.txt')):
            measurement = readEXLS3(os.path.join(path, fileOrDir))
        elif (fileOrDir.endswith('.csv')):
            measurement = readBonsai(os.path.join(path, fileOrDir))
        if (measurement is not None):
            measurementLocation = mapFileNameToLocation(fileOrDir)
            measurement = tagColumnNames(measurement, '_' + measurementLocation)
            measurements[measurementLocation] = measurement
    return measurements<jupyter_output><empty_output><jupyter_text>Calibration<jupyter_code>zeroMovementWindowSize = 200 # 10ms * zeroMovement

def calibrate(series):
    zeroWindowIndex = series.abs().rolling(zeroMovementWindowSize).median().sort_values().index[0]
    zero = series.rolling(zeroMovementWindowSize).median().iloc[zeroWindowIndex]
    series -= zero<jupyter_output><empty_output><jupyter_text>Synchronize the sensor data<jupyter_code>numberOfJumps = 3
jumpBinSize = 50 # 10ms * jumpBinSize = time per bin; bundles neighbor values to avoid multiple amplitudes during same jump
jumpSequenceLength = 800 # 10 ms * jumpSequenceLength
relativeMaxThreshold = 7 / 12

def binMeasurement(measurement, binSize):
    absMeasurement = measurement
    return absMeasurement.groupby(pd.cut(absMeasurement.index, np.arange(absMeasurement.index[0], absMeasurement.index[len(absMeasurement) - 1], binSize))).max()

def findJumpingWindow(measurement):
    measurement = measurement.head(int(len(measurement) / 2)) # jumping should be in first half
    absMeasurement = measurement.abs()
    threshold = absMeasurement.max() * relativeMaxThreshold
    absMeasurement = absMeasurement.apply(lambda value: value if value >= threshold else 0)
    bins = binMeasurement(absMeasurement, jumpBinSize).reset_index().drop('index', axis='columns')
    upperBound = bins.rolling(int(jumpSequenceLength / jumpBinSize)).sum().iloc[:,0].sort_values(ascending=False).index[0]
    lowerBound = upperBound - int(jumpSequenceLength / jumpBinSize)
    upperBound *= jumpBinSize
    lowerBound *= jumpBinSize
    return max(lowerBound - 100, 0), min(upperBound + 100, len(measurement) - 1)

def getFirstJumpIndex(measurement):
    windowIndicies = findJumpingWindow(measurement)
    window = measurement[windowIndicies[0]:  windowIndicies[1]]
    threshold = window.max() * relativeMaxThreshold
    window = window.apply(lambda value: 1 if value >= threshold else 0)
    return window.loc[window == 1].index[0]
def alignSignals(dfX, dfY):
    return getFirstJumpIndex(dfX) - getFirstJumpIndex(dfY)

def alignAccelerationYWithRightFoot(measurements, location, axis):
    offset = alignSignals(
        measurements['foot-r']['accY_foot-r'], 
        measurements[location]['acc' + axis.upper() + '_' + location])
    measurements[location] = measurements[location].shift(offset, axis='index')<jupyter_output><empty_output><jupyter_text>Exercise detection<jupyter_code>zeroMovementThreshold = 1.5 # given in meters per second

def getNextBinaryBlock(series, startPosition, minSubsequentMovements, zeroMode=True):
    start = series[startPosition:]
    start = start.loc[lambda value: value == 0] if zeroMode else start.loc[lambda value: value == 1]
    if (len(start) == 0):
        raise ValueError
    start = start.index[0]
    iValue = start
    zeroCounter = 0
    while (iValue < len(series)):
        if (not series[iValue]):
            zeroCounter += 1
            iValue += 1
        elif (zeroCounter < minSubsequentMovements):
            return getNextBinaryBlock(series, iValue + 1, minSubsequentMovements)
        else:
            break
    return start, iValue - 1

def findAllNonZeroBlocks(series, startPosition, minSubsequentZeroMovements=200, minSubsequentNonZeroMovements=29, ignoreMinSubsequentNonZeroMovements=True):
    '''
    Finds all blocks of movement (expects a filtered list with 1s and 0s, gives back indices of 1-blocks).
    Thresholds:
    - minSubsequentZeroMovements: minimal length of zero blocks to interrupt movement blocks
    - minSubsequentNonZeroMovements: minimal length of movement blocks
    - ignoreMinSubsequentNonZeroMovements: if minSubsequentNonZeroMovements should be ignored
    '''
    blocks = []
    start = series[startPosition:][series == 1].index[0]
    while (start < len(series)):
        try:
            zeroStart, zeroEnd = getNextBinaryBlock(series, start, minSubsequentZeroMovements)
            if ((((zeroStart - 1) - start) > minSubsequentNonZeroMovements) or ignoreMinSubsequentNonZeroMovements):
                blocks.append((start, zeroStart - 1))
            start = zeroEnd + 1
        except ValueError:
            if ((((len(series) - 1) - start) > minSubsequentNonZeroMovements) or ignoreMinSubsequentNonZeroMovements):
                blocks.append((start, len(series) - 1))
            start = len(series)
    return blocks

def splitDataFrameIntoExercises(df, columnName):
    measurement = df[columnName]
    windowIndicies = findJumpingWindow(measurement)
    filteredByTH = measurement.abs().apply(lambda value: 1 if value > zeroMovementThreshold else 0)
    exerciseIntervals = findAllNonZeroBlocks(filteredByTH, windowIndicies[1])
    return list(map(lambda interval: df[interval[0] : interval[1]], exerciseIntervals))<jupyter_output><empty_output><jupyter_text>Stride Detection<jupyter_code>restingThreshold=2.5 # given in m/s
minRestingInterval = 25 # we are taking the resting intervals of the right foot to detect ends of strides
minMovementInterval = 5 # movementIntervals seperate the resting intervals, we are not looking for them

def findRestingBlocks(series):
    filteredByTH = series.abs().apply(lambda value: 1 if value < restingThreshold else 0).reset_index(drop=True)
    return findAllNonZeroBlocks(filteredByTH, 0, minSubsequentZeroMovements=minMovementInterval, minSubsequentNonZeroMovements=minRestingInterval, ignoreMinSubsequentNonZeroMovements=False)

def findFirstStride(series, nextStrides):
    firstRestingInterval = findRestingBlocks(series)[0]
    if ((nextStrides[0][0] - minRestingInterval) > firstRestingInterval[0]):
        return (firstRestingInterval[0], nextStrides[0][0])

def findStrideIntervals(series):
    restingIntervals = findRestingBlocks(series)
    strideIntervals = []
    for i in range(len(restingIntervals) - 1):
        if (restingIntervals[i][1] < restingIntervals[i+1][1]):
            strideIntervals.append((restingIntervals[i][1], restingIntervals[i+1][1]))
    return strideIntervals

def splitExerciseIntoStrides(df):
    measurement = df['accY_foot-r']
    otherFoot = df['accY_foot-l']
    strideIntervals = findStrideIntervals(measurement)
    # in case of complete first stride being present but starting with left foot,
    # take its start until first already measured stride
    firstStride = findFirstStride(otherFoot, strideIntervals)
    if (firstStride):
        strideIntervals = [firstStride] + strideIntervals
    splittedExercise = [df]
    splittedExercise += list(map(lambda interval: df[interval[0] : interval[1]], strideIntervals))
    return splittedExercise<jupyter_output><empty_output><jupyter_text>Normalize Strides<jupyter_code>normalizedStrideLength = 10

def interpolateStride(stride):
    difference = normalizedStrideLength - len(stride)
    return stride.append(pd.DataFrame([[0 for column in stride.columns]] * difference, columns=stride.columns), ignore_index=True)

def resampleStride(stride):
    absStride = stride
    return absStride.groupby(pd.cut(absStride.index, np.linspace(absStride.index[0], absStride.index[len(absStride) - 1], normalizedStrideLength + 1))).median()

def normalizeStride(stride):
    for column in stride:
        stride[column] += stride[column].min()
        if (stride[column].max() > 0):
            stride[column] /= stride[column].max()

def normalizeStrides(strides):
    '''
    bring strides to same length by interpolating strides that are too short and resampling strides that are too long
    expects a list of stride dataframes
    '''
    for i, stride in enumerate(strides):
        if (len(stride) > normalizedStrideLength):
            strides[i] = resampleStride(stride)
        elif (len(stride) < normalizedStrideLength):
            strides[i] = interpolateStride(stride)
        normalizeStride(strides[i])
    return strides<jupyter_output><empty_output><jupyter_text>Combine date with calibration and sync<jupyter_code>minExerciseLength = 300 # 10ms * minExerciseLength
expectedExerciseCount = 6

def alignAll(measurements):
    alignAccelerationYWithRightFoot(measurements, 'hip-r', 'y')
    alignAccelerationYWithRightFoot(measurements, 'hip-l', 'y')
    alignAccelerationYWithRightFoot(measurements, 'foot-l', 'y')
    alignAccelerationYWithRightFoot(measurements, 'knee-l', 'y')
    alignAccelerationYWithRightFoot(measurements, 'knee-r', 'Y')
    
def calibrateAll(measurements):
    for location in measurements.values():
        for column in location.columns:
            calibrate(location[column])
            
def resetTimePointZero(mergedDf):
    firstIndex = max([mergedDf[column].first_valid_index() for column in mergedDf])
    lastIndex = min([mergedDf[column].last_valid_index() for column in mergedDf])
    return mergedDf[firstIndex:lastIndex]
        
def loadSyncedMeasurements(path):
    measurements = loadMeasurements(path)
    calibrateAll(measurements)
    alignAll(measurements)
    mergedDf = pd.DataFrame()
    for measurement in measurements.values():
        mergedDf = mergedDf.join(measurement, how='outer')
    mergedDf = resetTimePointZero(mergedDf).reset_index().drop('index', axis='columns')
    exercisesAndTurns = splitDataFrameIntoExercises(mergedDf, 'accY_foot-r')
    exercises = list(filter(lambda exerciseOrTurn: len(exerciseOrTurn) > minExerciseLength, exercisesAndTurns))
    if (len(exercises) is not expectedExerciseCount):
        print("Unexpected exercise count: ", len(exercises))
    data = [mergedDf] + exercises
    return data<jupyter_output><empty_output><jupyter_text>Classification  Similarity of Time Series  Dynamic Time Warping<jupyter_code>def DTWDistance(series1, series2):
    windowSize = 4
    DTW = {}
    
    windowSize = max(windowSize, abs(len(series1) - len(series2)))
    
    for i in range(-1, len(series1)):
        for j in range(-1, len(series2)):
            DTW[(i, j)] = float('inf')
    DTW[(-1, -1)] = 0
  
    for i in range(len(series1)):
        for j in range(max(0, i - windowSize), min(len(series2), i + windowSize)):
            dist = (series1[i] - series2[j])**2
            DTW[(i, j)] = dist + min(DTW[(i - 1, j)], DTW[(i, j - 1)], DTW[(i - 1, j - 1)])

    return math.sqrt(DTW[len(series1) - 1, len(series2) - 1])<jupyter_output><empty_output><jupyter_text>Classification algorithm  K-NN<jupyter_code>def collectVotesForStride(knnResults, strideIndex):
    strideVotingResults = []
    for key in knnResults.keys():
        cluster = knnResults[key]['results'][strideIndex]
        votingIndex = next((index for (index, vote) in enumerate(strideVotingResults) if vote['cluster'] == cluster), None)
        if (type(votingIndex) is int):
            strideVotingResults[votingIndex]['count'] += knnResults[key]['precision'][str(cluster)]
        else:
            strideVotingResults.append({'cluster': cluster, 'count': knnResults[key]['precision'][str(cluster)]})
    return strideVotingResults

def voteOnResults(knnResults):
    votingResults = []
    for strideIndex in range(len(knnResults[list(knnResults.keys())[0]]['results'])):
        strideVotes = collectVotesForStride(knnResults, strideIndex)
        mostFrequentVote = max(strideVotes, key=lambda x:x['count'])
        votingResults.append(mostFrequentVote['cluster'])
    return votingResults

def trainKnnForSensor(trainingData, sensorKey):
    targetClasses = [example[-1] for example in trainingData]
    cleanData = np.array(example[:-1] for example in trainingData)
        
    model = KNeighborsClassifier(n_neighbors=5, weights='distance', metric=DTWDistance)
    model.fit(trainingData, targetClasses)
    with open(os.path.join(modelPath, sensorKey + '.txt'), 'wb') as file:
         pickle.dump(model, file)

def trainKnn(trainStrides):
    try:
        shutil.rmtree(modelPath)
    except OSError:
        pass
    os.mkdir(modelPath)

    for i, key in enumerate(trainStrides):
        trainKnnForSensor(trainStrides[key], key)
        print('trained knn for sensor ', i+1 ,' of ', len(trainStrides.keys()), ' : ', key)
    print()
    print('finished training')
    
def testKnnForSensor(testData, sensorKey):
    with open(os.path.join(modelPath, sensorKey + '.txt'), 'rb') as file:
         model = pickle.load(file)
    
    testClasses = [example[-1] for example in testData]
    cleanData = np.array(example[:-1] for example in testData)

    results = model.predict(testData)
    report = classification_report(testClasses, results, output_dict=True)
    print('accuracy of', sensorKey, ': ')
    print(report)
    if 'accuracy' in report: del report['accuracy']
    return {
        'results': results,
        'precision': {classLabel: report[classLabel]['precision'] for classLabel in report}
    }

def testKnn(testStrides):
    if not os.path.isdir(modelPath):
        raise Error('There is no trained model.')

    knnResults = {key:[] for key in testStrides.keys()}

    for i, key in enumerate(testStrides):
        knnResults[key] = testKnnForSensor(testStrides[key], key)
        print('tested knn for sensor ', i+1 ,' of ', len(testStrides.keys()), ' : ', key)
    print()
    print('finished testing')
    print()
    votingResults = voteOnResults(knnResults)
    print('calculated votingResults')
    return classification_report(testStrides[list(testStrides.keys())[0]][:,-1],votingResults)<jupyter_output><empty_output><jupyter_text>Prepare Data for ClassificationExpected classification data input:  arrays of time series for train and test.Since we have multiple time series per stride (multiple sensors), we will put the time series into a dict with sensor keys and one array with all series each. The order is thus important and has to stay consistent to identify complete strides again.A majority vote will be performed on the sensors in the end to classify the strides.  Format Data and Split Training and Testing<jupyter_code>trainRatio = 0.7 # share of data that should go into training. e.g. 0.7: 70% training, 30% testing

def initializeSensorDict(strides):
    return {column:[] for column in strides['normal'][0]}

def listDictToNumpyArrayDict(dictionary):
    for key in dictionary:
        dictionary[key] = np.array(dictionary[key])
    return dictionary

def createSensorNumpyArray(stride, sensor, exerciseNumber):
    clusterLabel = float(exerciseNumber + 1)
    strideSensorWithLabel = stride[sensor].append(pd.Series([clusterLabel]), ignore_index=True)
    return np.array(strideSensorWithLabel)

def shuffleStrides(stridesDict):
    for exercise in stridesDict:
        random.shuffle(stridesDict[exercise])
    return stridesDict

def getTrainAndTestStrides(labelledStrides):
    train = initializeSensorDict(labelledStrides)
    test = initializeSensorDict(labelledStrides)
    
    labelledStrides = shuffleStrides(labelledStrides)

    for exerciseNumber, exercise in enumerate(labelledStrides):
        trainEndIndex = math.floor(len(labelledStrides[exercise]) * trainRatio)
        for stride in labelledStrides[exercise][:trainEndIndex]:
            for sensor in stride.columns:
                train[sensor].append(createSensorNumpyArray(stride, sensor, exerciseNumber))
        for stride in labelledStrides[exercise][trainEndIndex:]:
            for sensor in stride.columns:
                test[sensor].append(createSensorNumpyArray(stride, sensor, exerciseNumber))
    
    train = listDictToNumpyArrayDict(train)
    test = listDictToNumpyArrayDict(test)
    
    return train, test

def saveTrainTestSplit(labelledStrides):
    trainStrides, testStrides = getTrainAndTestStrides(labelledStrides)

    # most secure way to ensure that files are deleted if extant
    try:
        os.remove(trainingDataPath)
    except OSError:
        pass
    try:
        os.remove(testingDataPath)
    except OSError:
        pass
    with open(trainingDataPath, 'wb') as file:
         pickle.dump(trainStrides, file)
    with open(testingDataPath, 'wb') as file:
         pickle.dump(testStrides, file)<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>def loadData():
    labelledStrides = {
        'normal': [],
        'pelvic displacement': [],
        'limping': [],
        'shuffling': [],
        'small steps': [],
        'insecure walking': []
    }

    subjectPaths = [folderTuple[0] for folderTuple in os.walk(dataPath)][1:]

    for subjectPath in subjectPaths:
        print(subjectPath)
        loadedMeasurement = loadSyncedMeasurements(subjectPath)
        if (len(loadedMeasurement) == expectedExerciseCount + 1):
            for i, exercise in enumerate(loadedMeasurement[1:]):
                strides = splitExerciseIntoStrides(exercise)[1:]
                print(list(labelledStrides.keys())[i], len(strides))
                labelledStrides[list(labelledStrides.keys())[i]] += normalizeStrides(strides)
    return labelledStrides<jupyter_output><empty_output><jupyter_text>Execution We are persisting the training and testing data to prevent the data loading process having to be performed multiple times when testing different clustering parameters.**PERFORMING THE FOLLOWING CELL IS ONLY NECESSARY, IF TESTING AND TRAINING DATA SHOULD BE UPDATED / NEWLY SAVED**<jupyter_code>if runLoadData:
    saveTrainTestSplit(loadData())<jupyter_output><empty_output><jupyter_text>Cluster Data We are persisting the trained model to prevent the training process having to be performed multiple times when testing different clustering parameters.**PERFORMING THE FOLLOWING CELL IS ONLY NECESSARY, IF THE TRAINED MODEL SHOULD BE UPDATED / NEWLY SAVED**<jupyter_code>if runTrainModel:
    with open(trainingDataPath, 'rb') as file:
         trainStrides = pickle.load(file)

    trainKnn(trainStrides)
if runTestModel:
    with open(testingDataPath, 'rb') as file:
         testStrides = pickle.load(file)
    print(testKnn(testStrides))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11157287439495428, 'fraction_numerical': 0.008849557522123894, 'mean_word_length': 4.231202046035806, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '634030', 'license': 'Apache-2.0', 'n_tokens_mistral': 6436, 'n_tokens_neox': 6201, 'n_words': 1449, 'path': 'notebooks/Gait Problem Classification.ipynb', 'repo_name': 'dkipping/gait-analysis', 'size': '20453'}