<jupyter_start><jupyter_text>Import Libraries<jupyter_code>from __future__ import print_function
import neptune
import os, sys, math
import matplotlib.pyplot as plt
root   = os.path.abspath(os.getcwd())
libs   = root + '/libs'
data   = root + '/data'
sys.path.append(libs)
sys.path.append(data)
from nn_utils import *
from utilsData import *
from utilsData import read_dataset
from utilsData import transform_labels
import keras
from keras.utils import plot_model
from keras.models import Sequential, load_model
from keras.models import Model
from keras.datasets import mnist
from keras.layers import *
from keras.utils import to_categorical
import pandas as pd
import json
import numpy as np
import ctypes
import sklearn 
%matplotlib inline
import plotly.graph_objects as go
import time<jupyter_output><empty_output><jupyter_text>Define Constants and Variables<jupyter_code># Define Path Constants
dataRoot              = "data"
dataset_name          = 'Cloth'
rootWithSlash         = dataRoot + '/' 
dataTest              = dataset_name + "_TEST.txt"
dataTrain             = dataset_name + "_TRAIN.txt"

# Skript Constants
INTERPOLATION         = 2
NUMBER_OF_SAMPLES_CNN = 250
ADM_LOWER_LIMIT_INT_8 = 127
ADM_UPPER_LIMIT_INT_8 = ADM_LOWER_LIMIT_INT_8
MAX_NUMBER_OF_DATA_POINTS = 2927
budget                = 100
alpha                 = 0.5
window                = 10

pickleObjectNameW      = '/lowstandardhighvolumetipswater.pkl'
pickleObjectNameG      = '/lowstandardhighvolumetipsglycerin.pkl'<jupyter_output><empty_output><jupyter_text>Load Acquired Aspiration Data<jupyter_code># Load data from pickle Objects
sql_DF_Water          = pd.read_pickle(dataRoot+pickleObjectNameW)
sql_DF_Glycerol       = pd.read_pickle(dataRoot+pickleObjectNameG)
sql_DF                = pd.concat([sql_DF_Glycerol, sql_DF_Water])

# Select just pressure datas 
sql_DFFilteredAsp        = sql_DF.loc[sql_DF.loc[:,'J_Read Pressure Data Active'] == 1,:]
# Select just class ids and pressure datas 
sql_DFFilteredAspColumns = sql_DFFilteredAsp.iloc[:,-MAX_NUMBER_OF_DATA_POINTS:]
# Add Class id to dataframe
sql_DFFilteredAspColumns['ClassId'] = sql_DFFilteredAsp.loc[:,'ClassId']
# Reset index
sql_DFFilteredAspColumns = sql_DFFilteredAspColumns.reset_index()
sql_DFFilteredAspColumns = sql_DFFilteredAspColumns.drop(['index'], axis=1)<jupyter_output><empty_output><jupyter_text>Different Aspiration Curves Raw Data<jupyter_code>plt.close()
fig = go.Figure()

title = 'Different Aspiration Curves Raw Data'
# Add traces

for i in range(0, sql_DFFilteredAspColumns.shape[0]):
    # Normal Aspiration
    if int(sql_DFFilteredAspColumns.loc[i,'ClassId']) == 0:
        fig.add_trace(go.Scatter(x=np.arange(len(np.array(sql_DFFilteredAspColumns.iloc[i,:]))), y=np.array(sql_DFFilteredAspColumns.iloc[i,:]),name='Asp. Normal', 
                                 line=dict(color='rgb(0,0,0)', width=1)))
    # Air Aspiration
    elif int(sql_DFFilteredAspColumns.loc[i,'ClassId']) == 1:
        fig.add_trace(go.Scatter(x=np.arange(len(np.array(sql_DFFilteredAspColumns.iloc[i,:]))), y=np.array(sql_DFFilteredAspColumns.iloc[i,:]),name='Asp. Air', 
                                 line=dict(color='rgb(255,0,0)', width=1)))       
    # Cloth Aspiration
    elif int(sql_DFFilteredAspColumns.loc[i,'ClassId']) == 2:
        fig.add_trace(go.Scatter(x=np.arange(len(np.array(sql_DFFilteredAspColumns.iloc[i,:]))), y=np.array(sql_DFFilteredAspColumns.iloc[i,:]),name='Asp. Cloth', 
                                 line=dict(color='rgb(0,0,255)', width=1)))  
        
        
annotations = []        
        
annotations.append(dict(xref='paper', yref='paper', x=0.0, y=1.05,
                              xanchor='left', yanchor='bottom',
                              text=title,
                              font=dict(family='Arial',
                                        size=30,
                                        color='rgb(37,37,37)'),
                              showarrow=False))        

fig.update_layout(annotations=annotations,
                   yaxis_title='Pressure [pa]',
                   xaxis_title='Number of Samples [N] ')

fig.show()<jupyter_output><empty_output><jupyter_text>Scale Time Data to 250 PointsTwo different pressure data cases are possible - Aspiration pressure curve contains less than 250 points -> upsampling case - Aspiration pressure curve contains more than 250 points -> Downsampling case The idea for this problem is to transform a digital signal to a nearly continuous signal by interpolation and resample it such that the whole curve in time contains 250 samples.  Linear Interpolation of Curve<jupyter_code>sql_DFFilteredAspColumnsCopy                = sql_DFFilteredAspColumns.copy()
sql_DFFilteredAspColumnsCopy                = sql_DFFilteredAspColumnsCopy.drop(['ClassId'], axis= 1)
newDataframe                                = sql_DFFilteredAspColumnsCopy
for interpol in range(0,INTERPOLATION):
    newDataframe                            = pd.DataFrame()
    for i in range(0, sql_DFFilteredAspColumns.shape[0]):
        sql_DFFilteredAspColumnsWithoutNan  = np.array(sql_DFFilteredAspColumnsCopy.iloc[i,:].dropna())
        dataNumberingArray                  = np.arange(len(sql_DFFilteredAspColumnsWithoutNan)) + 1
        zeroPaddedData                      = np.insert(sql_DFFilteredAspColumnsWithoutNan, dataNumberingArray, 0)
        x = 0
        while x < (len(zeroPaddedData) - 2):
            zeroPaddedData[x + 1]           = ((zeroPaddedData[x] - zeroPaddedData[x + 2]) / 2.0) + zeroPaddedData[x + 2]
            x = x + 2
    
        interpolatedData                    = pd.DataFrame(zeroPaddedData)
        interpolatedDataT                   = interpolatedData.T
        
        newDataframe                        = pd.concat([newDataframe, interpolatedDataT])
        
    newDataframe                            = newDataframe.reset_index()
    newDataframe                            = newDataframe.drop(['index'], axis=1)
                                                    
    sql_DFFilteredAspColumnsCopy            = newDataframe<jupyter_output><empty_output><jupyter_text>Resample Curves Such That Wohle Curve Is Sampled Equally<jupyter_code>totalResampledData                          = pd.DataFrame()
for i in range(0, sql_DFFilteredAspColumns.shape[0]):
    sampleLengthInterpolatedCurve           = len(sql_DFFilteredAspColumnsCopy.iloc[i, :].dropna())
    numberOfOffsetSamples                   = sampleLengthInterpolatedCurve/NUMBER_OF_SAMPLES_CNN
    sampleIteration                         = 0
    newDataframeWithoutNan                  = sql_DFFilteredAspColumnsCopy.iloc[i, :].dropna()
    resampledData                           = []
    for k in range(0, NUMBER_OF_SAMPLES_CNN):
        resampledData.append(newDataframeWithoutNan[int(sampleIteration)])
        sampleIteration                    +=numberOfOffsetSamples
    
    resampledDataPd                         = pd.DataFrame(np.array(resampledData))
    totalResampledData                      = pd.concat([totalResampledData, resampledDataPd.T])
  
totalResampledData                          = totalResampledData.reset_index()
totalResampledData                          = totalResampledData.drop(['index'], axis=1)
sql_DFFilteredAspColumnsResIndex            = sql_DFFilteredAspColumns.reset_index()
sql_DFFilteredAspColumnsResIndex            = sql_DFFilteredAspColumnsResIndex.drop(['index'], axis=1)
totalResampledData['ClassId']               = list(sql_DFFilteredAspColumnsResIndex.loc[:,'ClassId'])
# Make sure ClassId stands at first position
totalResampledDataColumns                   = totalResampledData.columns.tolist()
totalResampledDataColumns                   = totalResampledDataColumns[-1:] + totalResampledDataColumns[:-1]
totalResampledData                          = totalResampledData[totalResampledDataColumns]<jupyter_output><empty_output><jupyter_text>Save to CSV<jupyter_code>totalResampledData.to_csv(rootWithSlash+dataTest, index=False, header=False)
totalResampledData.to_csv(rootWithSlash+dataTrain, index=False, header=False)<jupyter_output><empty_output><jupyter_text>Convert Test/Train Cap Level Json File to Dataframe and Numpy Array<jupyter_code># Load text file
datasets_dict = read_dataset(dataRoot, dataset_name) 
x_train       = datasets_dict[dataset_name][0]
y_train       = datasets_dict[dataset_name][1]
x_test        = datasets_dict[dataset_name][2]
y_test        = datasets_dict[dataset_name][3]
y_trainClass  = datasets_dict[dataset_name][1]
y_testClass   = datasets_dict[dataset_name][3]

nb_classes = len(np.unique(np.concatenate((y_train,y_test),axis =0)))
# make the min to zero of labels
y_train,y_test = transform_labels(y_train,y_test)

# save orignal y because later we will use binary
y_true = y_test.astype(np.int64) 
# transform the labels from integers to one hot vectors
enc = sklearn.preprocessing.OneHotEncoder()
enc.fit(np.concatenate((y_train,y_test),axis =0).reshape(-1,1))
y_train = enc.transform(y_train.reshape(-1,1)).toarray()
y_test = enc.transform(y_test.reshape(-1,1)).toarray()

if len(x_train.shape) == 2: # if univariate 
    # add a dimension to make it multivariate with one dimension 
    x_train = x_train.reshape((x_train.shape[0],x_train.shape[1],1))
    x_test = x_test.reshape((x_test.shape[0],x_test.shape[1],1))

input_shape = x_train.shape[1:]<jupyter_output><empty_output><jupyter_text>Scale Aspiration Data\begin{align}\Delta_s = max(S(t)) - min(S(t)) \\S_s(t) = \frac{\Delta_{s128}}{\Delta_s} \Bigg(S(t) - \bigg|\frac{|max(S(t))| - |min(S(t))|}{2}\bigg|\Bigg) \hspace{1cm} |max(S(t))| > |min(S(t))|\\S_s(t) = \frac{\Delta_{s128}}{\Delta_s} \Bigg(S(t) + \bigg|\frac{|max(S(t))| - |min(S(t))|}{2}\bigg|\Bigg) \hspace{1cm} |min(S(t))| > |max(S(t))|\end{align}where $S_s(t)$ defines the scaled pressure data, $S(t)$ defines the unscaled pressure data, $\Delta_{s128}$ the max range of a int_8 datatype (255) and $\Delta_s$ the max range of the unscaled pressure data $S(t)$<jupyter_code>plt.close()
fig = go.Figure()

title = 'Different Aspiration Curves Test Data'
# Add traces

for i in range(0,x_test.shape[0]):
    # Normal Aspiration
    if int(y_testClass[i]) == 0:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=x_test[i,:].ravel(),name='Asp. Normal', 
                                 line=dict(color='rgb(0,0,0)', width=1)))
    # Air Aspiration
    elif int(y_testClass[i]) == 1:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=x_test[i,:].ravel(),name='Asp. Air', 
                                 line=dict(color='rgb(255,0,0)', width=1)))       
    # Cloth Aspiration
    elif int(y_testClass[i]) == 2:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=x_test[i,:].ravel(),name='Asp. Cloth', 
                                 line=dict(color='rgb(0,0,255)', width=1)))  
        
        
annotations = []        
        
annotations.append(dict(xref='paper', yref='paper', x=0.0, y=1.05,
                              xanchor='left', yanchor='bottom',
                              text=title,
                              font=dict(family='Arial',
                                        size=30,
                                        color='rgb(37,37,37)'),
                              showarrow=False))        

fig.update_layout(annotations=annotations,
                   yaxis_title='Pressure [pa]',
                   xaxis_title='Number of Samples [N] ')

fig.show()<jupyter_output><empty_output><jupyter_text>Display Train X (Time) and Y (Normal-, Cloth-, Air-Aspiration)<jupyter_code>fig = go.Figure()
title = 'Different Aspiration Curves Train Data'
scaledTrain = x_train
# Add traces
for i in range(0,x_train.shape[0]):
    # Scale data
    temp        = []
    data        = x_train[i]
    delta_s     = max(data) - min(data)
    if(abs(max(data)) > abs(min(data))):
        tempScaled          = 253.0/delta_s * np.array(data) 
        scaledTrain[i,:]    = tempScaled + (ADM_UPPER_LIMIT_INT_8 - max(tempScaled))
    elif(abs(min(data)) > abs(max(data))): 
        tempScaled          = 253.0/delta_s * np.array(data) 
        scaledTrain[i,:]    = tempScaled + (-ADM_LOWER_LIMIT_INT_8 - min(tempScaled))
    
    # Normal Aspiration
    if int(y_trainClass[i]) == 0:
        fig.add_trace(go.Scatter(x=np.arange(len(x_train[i])), y=scaledTrain[i,:].ravel(),name='Asp. Normal', 
                                 line=dict(color='rgb(0,0,0)', width=1)))
    # Air Aspiration Aspiration
    elif int(y_trainClass[i]) == 1:
        fig.add_trace(go.Scatter(x=np.arange(len(x_train[i])), y=scaledTrain[i,:].ravel(),name='Asp. Air', 
                                 line=dict(color='rgb(255,0,0)', width=1)))       
    # Cloth Aspiration
    elif int(y_trainClass[i]) == 2 :
        fig.add_trace(go.Scatter(x=np.arange(len(x_train[i])), y=scaledTrain[i,:].ravel(),name='Asp. Cloth', 
                                 line=dict(color='rgb(0,0,255)', width=1)))  
        
annotations = []        
        
annotations.append(dict(xref='paper', yref='paper', x=0.0, y=1.0,
                              xanchor='left', yanchor='bottom',
                              text=title,
                              font=dict(family='Arial',
                                        size=30,
                                        color='rgb(37,37,37)'),
                              showarrow=False))        

fig.update_layout(annotations=annotations,
                   yaxis_title='Pressure [pa]',
                   xaxis_title='Number of Samples [N] ')
fig.show()<jupyter_output><empty_output><jupyter_text>Display Test X (Time) and Y (Normal-, Cloth-, Air-Aspiration)<jupyter_code>plt.close()
fig = go.Figure()

title = 'Different Aspiration Curves Test Data'
scaledTest = x_test
# Add traces
for i in range(0,x_test.shape[0]):
    # Scale data
    temp        = []
    data        = x_test[i]
    delta_s     = max(data) - min(data)
    if(abs(max(data)) > abs(min(data))):
        tempScaled          = 253.0/delta_s * np.array(data) 
        scaledTest[i,:]    = tempScaled + (ADM_UPPER_LIMIT_INT_8 - max(tempScaled))
    elif(abs(min(data)) > abs(max(data))): 
        tempScaled          = 253.0/delta_s * np.array(data) 
        scaledTest[i,:]    = tempScaled + (-ADM_LOWER_LIMIT_INT_8 - min(tempScaled))

for i in range(0,x_test.shape[0]):
    # Normal Aspiration
    if int(y_testClass[i]) == 0:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=scaledTest[i,:].ravel(),name='Asp. Normal', 
                                 line=dict(color='rgb(0,0,0)', width=1)))
    # Air Aspiration
    elif int(y_testClass[i]) == 1:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=scaledTest[i,:].ravel(),name='Asp. Air', 
                                 line=dict(color='rgb(255,0,0)', width=1)))       
    # Cloth Aspiration
    elif int(y_testClass[i]) == 2:
        fig.add_trace(go.Scatter(x=np.arange(len(x_test[i])), y=scaledTest[i,:].ravel(),name='Asp. Cloth', 
                                 line=dict(color='rgb(0,0,255)', width=1)))  
        
        
annotations = []        
        
annotations.append(dict(xref='paper', yref='paper', x=0.0, y=1.05,
                              xanchor='left', yanchor='bottom',
                              text=title,
                              font=dict(family='Arial',
                                        size=30,
                                        color='rgb(37,37,37)'),
                              showarrow=False))        

fig.update_layout(annotations=annotations,
                   yaxis_title='Pressure [pa]',
                   xaxis_title='Number of Samples [N] ')

fig.show()<jupyter_output><empty_output><jupyter_text>Define Layers of CNN Algorithm<jupyter_code>#https://adventuresinmachinelearning.com/keras-tutorial-cnn-11-lines/
def KModel(x_train, y_train, x_test, y_test):
    # Shuffle
    permutation = np.random.permutation(y_train.shape[0])
    x_train = x_train[permutation, :, :]
    y_train = y_train[permutation]
       
    inputs = Input(shape=x_train.shape[1:])
    # Layer Block 1
    x = Conv1D(32, kernel_size=(9), strides=(2), padding='same')(inputs)
    x = BatchNormalization()(x)
    x = Dropout(0.2)(x)
    x = ReLU()(x)
    x = MaxPool1D(2, strides=2)(x)

    # Layer Block 2
    x1 = Conv1D(32, kernel_size=(5), strides=(1), padding="same")(x)
    x1 = BatchNormalization()(x1)
    x1 = Dropout(0.2)(x1)
    x1 = ReLU()(x1)
    x1 = MaxPool1D(2, strides=2)(x1)

    # Layer Block 3
    x2 = Conv1D(32, kernel_size=(3), strides=(1), padding="same")(x)
    x2 = BatchNormalization()(x2)
    x2 = Dropout(0.2)(x2)
    x2 = ReLU()(x2)
    x2 = MaxPool1D(2, strides=2)(x2)

    # Layer Block 4
    x3 = MaxPool1D(2, strides=2)(x)
    x3 = Dropout(0.2)(x3)

    # concate all inception layers
    x = concatenate([ x1, x2,x3], axis=-1)

    # conclusion
    x = Conv1D(48, kernel_size=(3), strides=(1), padding="same")(x)
    x = ReLU()(x)
    x = MaxPool1D(2, strides=2)(x)
    x = Dropout(0.2)(x)

    # our netowrk is not that deep, so a hidden fully connected layer is introduce
    x = Flatten()(x)
    x = Dense(64)(x)
    x = Dropout(0.2)(x)
    x = ReLU()(x)
    x = Dense(3)(x)
    y = Softmax()(x)
    
    model = Model(inputs=inputs, outputs=y)
    model.compile(loss="categorical_crossentropy", optimizer="Adam", metrics=['accuracy'])
    model.fit(x_train, y_train, batch_size=16, epochs=100, shuffle=True, verbose=2, validation_data=(x_test, y_test))
    return model<jupyter_output><empty_output><jupyter_text>Train CNN Algorithm<jupyter_code>if os.path.exists('.shift_list'):
    os.remove(".shift_list")
model = KModel(scaledTrain, y_train, scaledTest, y_test)
model.save(dataRoot+'model.h5')<jupyter_output><empty_output><jupyter_text>Generate C Code for NN Library and Transfer Weights File SVN<jupyter_code>generate_model(model, x_test, name='NN_weights.h')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12887057251058143, 'fraction_numerical': 0.019269325016707507, 'mean_word_length': 2.7535535117056855, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 65, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '583168', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 6288, 'n_tokens_neox': 5977, 'n_words': 1010, 'path': 'phenomenDetection.ipynb', 'repo_name': 'achenbachsven/learningSkript', 'size': '17956'}