<jupyter_start><jupyter_text>Cutting the tests  PurposeCut the tests based on their states.  MethodologyLook at the velocities, accelerations and rudder signal to determine the good parts of the tests  Setup<jupyter_code># %load imports.py
%matplotlib inline
%load_ext autoreload
%autoreload 2

## External packages:
import pandas as pd
pd.options.display.max_rows = 999
pd.options.display.max_columns = 999
pd.set_option("display.max_columns", None)

import numpy as np
import os
import matplotlib.pyplot as plt

import plotly.express as px 
import plotly.graph_objects as go

import seaborn as sns
import sympy as sp
from sympy.physics.mechanics import (dynamicsymbols, ReferenceFrame,
                                      Particle, Point)
from sympy.physics.vector.printing import vpprint, vlatex
from IPython.display import display, Math, Latex
from src.substitute_dynamic_symbols import run, lambdify

import pyro

import sklearn
import pykalman
from statsmodels.sandbox.regression.predstd import wls_prediction_std
import statsmodels.api as sm

from scipy.integrate import solve_ivp
import seaborn as sns

## Local packages:
from src.data import mdl



df_runs = mdl.runs()
mask = ((~df_runs['sailing']) &
        (
            (df_runs['test_type'] == 'reference speed') |
            (df_runs['test_type'] == 'rodergrundvinkel') |
            ((df_runs['series_number'] == 5) & (df_runs['test_number'] == 1)  & (df_runs['run_number'] == 3)) |
            ((df_runs['series_number'] == 5) & (df_runs['test_number'] == 2)  & (df_runs['run_number'] == 6)) |
            ((df_runs['series_number'] == 5) & (df_runs['test_number'] == 3)  & (df_runs['run_number'] == 2)) |
            ((df_runs['series_number'] == 5) & (df_runs['test_number'] == 4)  & (df_runs['run_number'] == 1)) |
            ((df_runs['series_number'] == 5) & (df_runs['test_number'] == 5)  & (df_runs['run_number'] == 1)) 
            
       ))
df_runs_selected = df_runs.loc[mask].copy()
def load_run(id):
    
    df, units, meta_data = mdl.load(id=id, dir_path='../data/processed/kalman')
    df.index = df.index.total_seconds()
    df.index-=df.index[0]
    df.sort_index(inplace=True)
    df['-delta'] = -df['delta']
    df['V'] = np.sqrt(df['u']**2 + df['v']**2)
    
    return df, units, meta_data

df_all = pd.DataFrame()
for id,row in df_runs_selected.iterrows():
    
    df_, units, meta_data = load_run(id)
    df_['id'] = id
    df_['t'] = df_.index
    df_all = df_all.append(df_, ignore_index=True)

    
df_all['thrust'] = df_all['Prop/PS/Thrust'] + df_all['Prop/SB/Thrust']
df_all['U'] = df_all['V']

df_all_clean = df_all.copy()
df_all = pd.merge(left=df_all, right = df_runs_selected, how='left', left_on='id', right_index=True)
ref_runs = df_all.groupby(by='test_type').get_group('reference speed')
runs = ref_runs.groupby(by='id')
df_cut = pd.DataFrame()
for id, group in runs:
    
    df_rolling = group.rolling(window=500).std()
    mask = df_rolling['u1d'] < 0.0004
    df_ = group.loc[mask].copy()
    if len(df_) > 300:
        df_cut = df_cut.append(df_)
    
    
runs_cut = df_cut.groupby(by='id')
for id, group in runs:
    
    fig,ax=plt.subplots()
    fig.set_size_inches(20,3)
    
    meta_data = df_runs_selected.loc[id]
    title = f'id:{id} ({meta_data["test_type"]})'
    ax.set_title(title)
    
    key='u'
    group.plot(x='t', y=key,ax=ax)
    
    try: 
        group_cut = runs_cut.get_group(id)
    except:
        pass
    else:
        group_cut.iloc[[0]].plot(x='t', y=key, style='go', label='start', ax=ax)
        group_cut.iloc[[-1]].plot(x='t', y=key, style='ro', label='stop', ax=ax)
        
    ax.set_ylim(df_all[key].min(),df_all[key].max())
    ax.legend(loc='upper left')
    
    ax2 = ax.twinx()
    key='u1d'
    group.plot(x='t', y=key,ax=ax2, style='r-', title=title)
        
    ax2.set_ylim(df_all[key].min(),df_all[key].max())
    ax2.get_legend().set_visible(False)
    ax2.legend(loc='upper right')
    ax2.grid(True)
    
ref_rud_runs = df_all.groupby(by='test_type').get_group('rodergrundvinkel')
runs = ref_rud_runs.groupby(by='id')

for id, group in runs:
    
    df_rolling = group.rolling(window=500).std()
    mask = ((df_rolling['r'] < 0.0004) & (df_rolling['u1d'] < 0.0005))
    df_ = group.loc[mask].copy()
    
    df_ = group.loc[mask].copy()
    if len(df_) > 300:
        df_cut = df_cut.append(df_)
runs_cut = df_cut.groupby(by='id')
for id, group in runs:
    
    fig,ax=plt.subplots()
    fig.set_size_inches(20,3)
    
    meta_data = df_runs_selected.loc[id]
    title = f'id:{id} ({meta_data["test_type"]})'
    ax.set_title(title)
    
    key='u1d'
    group.plot(x='t', y=key,ax=ax)
    
    try: 
        group_cut = runs_cut.get_group(id)
    except:
        pass
    else:
        group_cut.iloc[[0]].plot(x='t', y=key, style='go', label='start', ax=ax)
        group_cut.iloc[[-1]].plot(x='t', y=key, style='ro', label='stop', ax=ax)
        
    ax.set_ylim(df_all[key].min(),df_all[key].max())
    ax.legend(loc='upper left')
    
    ax2 = ax.twinx()
    key='r'
    group.plot(x='t', y=key,ax=ax2, style='r-', title=title)
        
    ax2.set_ylim(df_all[key].min(),df_all[key].max())
    ax2.get_legend().set_visible(False)
    ax2.legend(loc='upper right')
    ax2.grid(True)
mask = ((df_all['test_type']=='reference speed') | 
        (df_all['test_type']=='rodergrundvinkel')
       )
df_man = df_all.loc[~mask]
for id,group in df_man.groupby(by='id'):
    df_ = group.iloc[0:-500].copy()  # "strange thing may happen in the end"
    df_cut = df_cut.append(df_)
df_cut['test_type'].unique()
df_cut<jupyter_output><empty_output><jupyter_text>Save<jupyter_code>save_dir = '../data/processed/kalman_cut'
if not os.path.exists(save_dir):
    os.mkdir(save_dir)

runs = df_all_clean.groupby(by='id')
for id, group in df_cut.groupby(by='id'):
    
    start_index = group.index[0]
    stop_index = group.index[-1]
    df = runs.get_group(id).loc[start_index:stop_index].copy()
    df.set_index('t', inplace=True)
    save_name = f'{id}.csv'
    save_path = os.path.join(save_dir,save_name)
    df.to_csv(save_path)
    
   
    
    
    
df.head()
start_index
stop_index
index = list(set(df_runs_selected.index) & set(df_cut['id'].unique()))
df_runs = df_runs_selected.loc[index].copy()
save_name = 'runs.csv'
save_path = os.path.join(save_dir,save_name)
df_runs.to_csv(save_path)
index<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16658852945772776, 'fraction_numerical': 0.013127051101734646, 'mean_word_length': 3.535790219702339, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '521303', 'license': 'MIT', 'n_tokens_mistral': 2592, 'n_tokens_neox': 2515, 'n_words': 425, 'path': 'notebooks/05.03_cutting.ipynb', 'repo_name': 'martinlarsalbert/wPCC', 'size': '6399'}