<jupyter_start><jupyter_text>Importing Data<jupyter_code>raw_data=pd.read_csv("../data/processed/gathered_w_rms.csv",index_col=False )

raw_data
cleaned_data=pd.read_csv("../data/processed/cleaned_w_interp.csv",index_col=False )
cleaned_data
def eliminate_discontinuity(df):
    # Extract individual walks
    subject_id=df['SubjectId'].values
    walks=list(df.drop_duplicates(subset=['walk'])['walk'].values)
    lags=[]
    for walk in walks:
        walk_data=df[df['walk']==walk]
        stamps=walk_data['stamp'].values
        diff=np.diff(stamps)
        time_lag=np.where(diff<0.015,0,diff )
        lag_idx=np.nonzero(time_lag)
        lags.append(lag_idx)
        if time_lag.sum()>0:
            # Drop data beyond the first discontinuity 
            walk_idx=np.array(walk_data.index)
            sliced_walk=walk_idx[lag_idx[0][0]:] #sliced from the first discontinuity to the end 
            df=df.drop(sliced_walk) #drop the sliced data from the df
    return df.reset_index(drop=True)

         
raw_data=eliminate_discontinuity(raw_data)
raw_data.to_csv("../data/processed/dataset_raw.csv", index=False)
raw_data

cleaned_data=eliminate_discontinuity(cleaned_data)
cleaned_data.to_csv("../data/processed/dataset_interp.csv", index=False)
cleaned_data<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12538461538461537, 'fraction_numerical': 0.006153846153846154, 'mean_word_length': 4.224899598393574, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '474236', 'license': 'MIT', 'n_tokens_mistral': 479, 'n_tokens_neox': 465, 'n_words': 59, 'path': 'notebooks/7.0-eliminate_data_discontinuity.ipynb', 'repo_name': 'm-patterson-wearable/gait_speed_prediction', 'size': '1300'}