<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount("/gdrive", force_remount = True)
%cd "/gdrive/My Drive/tensor-flow-state/tensor-flow-state"
import pandas as pd
df = pd.read_csv("data/combined_df.csv", index_col = 0, parse_dates = True)
cols = ["speed", "flow", "speed_-2", "speed_-1","speed_+1", "speed_+2", "flow_-2", "flow_-1", "flow_+1", "flow_+2", "speed_limit"]
speed_cols = ["speed", "speed_-2", "speed_-1","speed_+1", "speed_+2"]
flow_cols = ["flow", "flow_-2", "flow_-1", "flow_+1", "flow_+2"]
def nans(df, title = ""):
    records = len(df)
    null_records = len(df.loc[df.isnull().any(axis=1)])
    null_ratio = len(df.loc[df.isnull().any(axis=1)]) / len(df)

    print(f"\
    {title}\n\
    ---------------------------------\n\
    Number of records:      {records:,}\n\
    Number of null records:   {null_records:,}\n\
    ---------------------------------\n\
    Null ratio:                 {null_ratio:.3f}\
    ")
nans(df, title = "Status before imputation")
def fill_na_row_mean(df):
    row_avgs = df.mean(axis=1).values.reshape(-1,1)
    df = df.fillna(0) + df.isna().values * row_avgs
    return df
speed_df = fill_na_row_mean(df[speed_cols])
flow_df = fill_na_row_mean(df[flow_cols])
df = speed_df.join(flow_df, how = 'inner').join(df[['speed_limit']], how = 'inner')
nans(df, title = "Status after row meaning nans")
df.to_csv("data/df_NAs_row_meaned.csv")<jupyter_output><empty_output><jupyter_text>Now interpolate the rest with week shifting.<jupyter_code># Interpolate null vals for the first week of data of speed and flow cols
def interpolate_week(df, cols):
    week = 7 * 24 * 60
    for col in cols: 
        df.iloc[:week, df.columns.get_loc(col)] = df[col][:week].interpolate(method = 'time')
    return df

import numpy as np
# Replace remaining nulls with value from 1 week previous
def shift_week(df, cols):
    # Use RangeIndex for the this operation
    df['timestamp'] = df.index
    df.reset_index(drop = True, inplace = True)
    week = 7 * 24 * 60
    for col in cols:
        col_index = df.columns.get_loc(col)
        for row in df.itertuples():
            if np.isnan(row[col_index + 1]):
                df.iat[row[0], col_index] = df.iat[(row[0] - week), col_index]
    # Return to DateTimeIndex again
    df.set_index(pd.to_datetime(df.timestamp.values), inplace = True) 
    df.drop('timestamp', axis = 1, inplace = True)
    return df
df = interpolate_week(df, cols)
df = shift_week(df, cols)
nans(df, title = "Status after week shifting")
df.head()
df.to_csv("data/df_imputed_week_shift.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1820964039785769, 'fraction_numerical': 0.014919663351185922, 'mean_word_length': 3.424703891708968, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '285216', 'license': 'MIT', 'n_tokens_mistral': 975, 'n_tokens_neox': 947, 'n_words': 216, 'path': 'old_notebooks/3_data_impute.ipynb', 'repo_name': 'ppiont/tensor-flow-state', 'size': '2614'}