<jupyter_start><jupyter_text>Preprocess research<jupyter_code>import numpy as np
import scipy
from scipy.signal import butter, lfilter, lfilter_zi
import plotly.graph_objects as go
from plotly.subplots import make_subplots
def read_sample_data(path):
    with open(path, "r") as f:
        data = f.read()
    
    data = data.split(",")
    return np.array(data).astype(np.float32)
        
sample_data = read_sample_data("sample_data/raw_data.txt")
sample_prep_data = read_sample_data("sample_data/preprocessed_data.txt")
n_timesteps = 200 # 2 seconds at 100Hz
n_features = 3 # X, Y, Z
sample_data = sample_data.reshape(n_timesteps, n_features)
sample_prep_data = sample_prep_data.reshape(n_timesteps, n_features)
sample_prep_data = read_sample_data("sample_data/untitled.txt")
prepmic = sample_prep_data.reshape(n_timesteps, n_features)
np.where( np.abs(prepmic.flatten()) < 0.000001)<jupyter_output><empty_output><jupyter_text>Raw data<jupyter_code>fig = make_subplots(rows=1, cols=1, subplot_titles=("Raw data") )

axes_names = ["X", "Y", "Z"]
colors = ["red", "green", "blue"]

x = np.arange(0, 200, 1)

for feature, axis_name, color in zip(sample_data.T, axes_names, colors):
    fig.add_trace(go.Scatter(x=x, y=feature, mode='lines', name=axis_name, opacity=0.7,
                 line =dict(color=color)),
                 row=1, col=1)
    
fig.show()<jupyter_output><empty_output><jupyter_text>Python preprocessing<jupyter_code>def butter_lowpass(lowcut, fs, order=5):
    nyq = 0.5 * fs
    low = lowcut / nyq
    b, a = butter(order, low, btype='lowpass', output='ba')
    return b, a


def butter_lowpass_filter(data, lowcut, fs, order=5):
    b, a = butter_lowpass(lowcut, fs, order=order)
    #zi = lfilter_zi(b, a)
    #y, _ = lfilter(b, a, data, zi = zi*data[0])
    y = lfilter(b, a, data)
    return y
sampling_freq = 100
filter_order = 2
cutoff = 20
# Using libraries
def preprocess_high_level(data):
    data = data - data.mean(axis=0)
    
    data = data.T
    for i, ax in enumerate(data): # I could not vectorize it properly
        data[i] = butter_lowpass_filter(ax, cutoff, sampling_freq, filter_order)
    
    return data.T<jupyter_output><empty_output><jupyter_text>We need the filter coefficents to implement our own discrete filter:<jupyter_code>def print_tf(num, den, mode='z'):
    num_str = " + ".join(["{:.4f}{}^{}".format(el, mode, exp) for el, exp in zip(num.flatten(), range(len(num.flatten())-1, -1, -1)) if abs(el) > 1e-6])
    den_str = " + ".join(["{:.4f}{}^{}".format(el, mode, exp) for el, exp in zip(den.flatten(), range(len(den.flatten())-1, -1, -1)) if abs(el) > 1e-6])
    max_len = max(len(num_str), len(den_str))
    tf_str = " "*((max_len-len(num_str))//2) + num_str +"\n" + "-"*max_len+"\n" + " "*((max_len-len(den_str))//2) + den_str
  
    print(tf_str)
b, a = butter_lowpass(cutoff, sampling_freq, filter_order)
print_tf(b, a)
b0, b1, b2 = b.flatten()
a0, a1, a2 = a.flatten()
# Now something more similar to what the C code will look like
def preprocess_mid_level(data):
    data = data - data.mean(axis=0)
    
    # C style filter
    xk1 = xk2 = yk1 = yk2 = 0
    x = data
    
    y_filterc = []
    for i in range(len(x)):
        xk = x[i]
        yk = (b0*xk + b1*xk1 + b2*xk2 - a1*yk1 - a2*yk2)

        yk2 = yk1
        yk1 = yk
        xk2 = xk1
        xk1 = xk

        y_filterc.append(yk)
    
    data = np.array(y_filterc)
    return data<jupyter_output><empty_output><jupyter_text>Comparison<jupyter_code>data_prep_high_level = preprocess_high_level(sample_data)
data_prep_mid_level = preprocess_mid_level(sample_data)
fig = make_subplots(rows=1, cols=1, subplot_titles=("Raw data") )

axes_names = ["X", "Y", "Z"]
colors = ["red", "green", "blue"]

x = np.arange(0, 200, 1)

for feature, axis_name, color in zip(sample_data.T, axes_names, colors):
    fig.add_trace(go.Scatter(x=x, y=feature, mode='lines', name=axis_name, opacity=0.7,
                 line =dict(color=color)),
                 row=1, col=1)
    
fig.show()
fig = make_subplots(rows=3, cols=1, 
                    subplot_titles=("Raw data", "Using scipy", "Manual implementation"),
                    vertical_spacing=0.06)

axes_names = ["X", "Y", "Z"]
colors = ["red", "green", "blue"]

x = np.arange(0, 200, 1)

for feature, axis_name, color in zip(sample_data.T, axes_names, colors):
    fig.add_trace(go.Scatter(x=x, y=feature, mode='lines', name=axis_name, opacity=0.7,
                 line =dict(color=color)),
                 row=1, col=1)

for feature, axis_name, color in zip(data_prep_high_level.T, axes_names, colors):
    
    fig.add_trace(go.Scatter(x=x, y=feature, mode='lines', name=axis_name, opacity=0.7,
                 line =dict(color=color), showlegend=False),
                 row=2, col=1)

for feature, axis_name, color in zip(data_prep_mid_level.T, axes_names, colors):
    fig.add_trace(go.Scatter(x=x, y=feature, mode='lines', name=axis_name, opacity=0.7,
                 line =dict(color=color), showlegend=False),
                 row=3, col=1)

fig.update_layout(width = 800, height = 1000)
fig.update_xaxes(matches='x')
fig.show()
# Save fig for documentation
import pickle

with open("preprocessing_comparison.pkl", "wb") as f:
    pickle.dump(fig, f)

fig = make_subplots(rows=1, cols=1, subplot_titles=("Raw data") )

axes_names = ["X", "Y", "Z"]
colors = ["red", "green", "blue"]

x = np.arange(0, 200, 1)

for feature, axis_name, color in zip(sample_prep_data.T, axes_names, colors):
    fig.add_trace(go.Scatter(x=x, y=feature, mode='lines', name=axis_name, opacity=0.7,
                 line =dict(color=color)),
                 row=1, col=1)
    
fig.show()<jupyter_output><empty_output><jupyter_text>Looks good, the only difference is in the initial conditions.  Checking the model has the same output with the C-like preprocessing Correct label: K<jupyter_code>import tensorflow as tf

# Load the TFLite model and allocate tensors.
interpreter = tf.lite.Interpreter(model_path="sample_data/model.lite")
interpreter.allocate_tensors()

# Get input and output tensors.
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
# Test the model on random input data.
input_data = np.expand_dims(data_prep_mid_level.flatten(), axis=0)
interpreter.set_tensor(input_details[0]['index'], input_data)

interpreter.invoke()

# The function `get_tensor()` returns a copy of the tensor data.
# Use `tensor()` in order to get a pointer to the tensor.
output_data = interpreter.get_tensor(output_details[0]['index'])
output_data
np.set_printoptions(precision=4)
np.set_printoptions(suppress=True)
print(data_prep_mid_level.flatten())<jupyter_output>[-0.0283  0.0061  0.0812 -0.0903  0.0308  0.2767 -0.114   0.0622  0.4134
 -0.0859  0.073   0.4064 -0.0622  0.0597  0.3416 -0.0332  0.0252  0.2662
  0.0342  0.0007  0.1803  0.1142  0.0263  0.0913  0.1613  0.077   0.0159
  0.1659  0.12   -0.0227  0.1582  0.1646 -0.016   0.1599  0.2293  0.0452
  0.1566  0.3011  0.155   0.1268  0.3229  0.237   0.0747  0.2551  0.2414
  0.0254  0.1431  0.2087 -0.0066  0.0713  0.1893 -0.0253  0.0684  0.1658
 -0.0384  0.0609  0.1196 -0.058  -0.0301  0.1064 -0.0915 -0.1421  0.1516
 -0.119  -0.1924  0.2109 -0.1246 -0.1821  0.2433 -0.1077 -0.1292  0.2313
 -0.0679 -0.0513  0.1777 -0.0242  0.0292  0.1395 -0.0052  0.0865  0.1836
 -0.0136  0.1119  0.2667 -0.0257  0.1116  0.2992 -0.0213  0.0807  0.2778
 -0.0086  0.0397  0.2473 -0.0194  0.018   0.2224 -0.0682 -0.0186  0.2251
 -0.1209 -0.086   0.2564 -0.1343 -0.1247  0.2516 -0.1112 -0.092   0.1953
 -0.0723 -0.0132  0.1296 -0.0328  0.0516  0.0875 -0.0023  0.0816  0.0979
  0.025   0.1112  0.1666  0.036   0.1476  0.2493  0[...]
{'fraction_non_alphanumeric': 0.14797446919369545, 'fraction_numerical': 0.09600104207372671, 'mean_word_length': 3.443287037037037, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '167592', 'license': 'MIT', 'n_tokens_mistral': 3459, 'n_tokens_neox': 3104, 'n_words': 672, 'path': 'applications/handwriting_recognition/research/custom_processing_block_ei/preprocess_research.ipynb', 'repo_name': 'PHANzgz/embedded-deep-learning', 'size': '7677'}