<jupyter_start><jupyter_text>Reading Data<jupyter_code>import numpy as np
from matplotlib import pyplot as plt
from collections import namedtuple
from collections import deque
from slurm import storage, files
from pprint import pprint
from colorama import Fore
import datetime
TempPres = namedtuple("TempPres", "temperature pressure")
Vector2 = namedtuple("Vector2", "x y")
Vector3 = namedtuple("Vector3", "x y z")
Lidar = namedtuple("Lidar","distance")
ImuSoxLis = namedtuple("ImuSoxLis", "ax ay az wx wy wz temperature mx my mz")
def toNumpy(data, key):
    return np.array([x[key] for x in data])

def imuToNumpy(data):
    a = np.array([x["soxlis"][0:3] for x in data])
    g = np.array([x["soxlis"][3:6] for x in data])
    t = np.array([x["soxlis"][6:7] for x in data])
    m = np.array([x["soxlis"][7:] for x in data])
    return a,g,m,t<jupyter_output><empty_output><jupyter_text>Data Collection<jupyter_code>data = storage.read("data.pickle")
for k,v in data[0].items():
    print(f"{Fore.BLUE}{k}: {Fore.YELLOW}{v}{Fore.RESET}")
accel, gyro, mag, temp = imuToNumpy(data)
tp = toNumpy(data, "dps310")
lidar = toNumpy(data, "lidar")

# change to 0 start
ts = toNumpy(data, "timestamp")
tss = ts[0]
ts = np.array([x - tss for x in ts])
plt.figure(figsize=(20,10))

plt.subplot(2,2,1)
plt.plot(accel)
plt.grid(True)
plt.title("Accel");

plt.subplot(2,2,2)
plt.plot(gyro)
plt.grid(True)
plt.title("Gyro");

plt.subplot(2,2,3)
plt.plot(mag)
plt.grid(True)
plt.title("Mag");

plt.subplot(2,2,4)
plt.plot(ts,temp)
plt.grid(True)
plt.title("IMU Temperature [C]");
plt.figure(figsize=(20,10))
plt.subplot(1,2,1); plt.plot(ts,tp[:,0]); plt.grid(True); plt.title("Temp [C]")
plt.subplot(1,2,2); plt.plot(ts,tp[:,1]); plt.grid(True); plt.title("Pressure [kPa]");
plt.figure(figsize=(20,10))
plt.plot(ts[1:], 1/np.diff(ts))
plt.title("Sample Rate [Hz]")
plt.grid(True);
# lidar = [x if x < 100 else 0 for x in lidar]

plt.figure(figsize=(20,10))
plt.plot(ts, lidar)
plt.title("LIDAR [cm]")
plt.grid(True);<jupyter_output><empty_output><jupyter_text>SaveNow that the data is checked and appears to be good, save a copy of the parsed data.<jupyter_code>sdata = {
    "accel": accel,
    "mag": mag,
    "gyro": gyro,
    "imutemp": temp,
    "temp": tp[:,0],
    "press": tp[:,1],
    "lidar": lidar,
    "ts": ts
}

storage.write(f"parsed-data-{datetime.date.today().isoformat()}.pickle", sdata)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1896694214876033, 'fraction_numerical': 0.025206611570247933, 'mean_word_length': 5.525606469002695, 'pattern_counts': {'":': 8, '<': 14, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '104762', 'license': 'MIT', 'n_tokens_mistral': 1047, 'n_tokens_neox': 1007, 'n_words': 222, 'path': 'quadcopter-sensors-ins/python/read-data.ipynb', 'repo_name': 'MomsFriendlyRobotCompany/mjolnir', 'size': '2420'}