<jupyter_start><jupyter_text>Feature Engineering: Use timestamp and lastStart to create timeSinceLastStart<jupyter_code>def datetime_parser(datetime_str):
    try:
        date_str, time_str = datetime_str.split("T")
    except:
        return 
    time_str=time_str[:8]
    year, month, day = date_str.split("-")
    hour, minut, sec = time_str.split(":")
    return dt.datetime(int(year), int(month), int(day), int(hour), int(minut), int(sec))
        

def time_diff_in_minutes(dt_0, dt_1):
    return np.round((dt_1 - dt_0).total_seconds() / 60.0, 0)
df.timestamp = df.timestamp.apply(datetime_parser)
df.lastStart = df.lastStart.apply(datetime_parser)

df['timeSinceLastStart'] = df.apply(lambda row: time_diff_in_minutes(row['lastStart'], row['timestamp']), axis=1).fillna(0) 
print(df.head())
print(df.shape)
df.to_pickle('../data/preprocessed_training_data.pkl')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1511111111111111, 'fraction_numerical': 0.012222222222222223, 'mean_word_length': 5.256944444444445, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '623150', 'license': 'MIT', 'n_tokens_mistral': 329, 'n_tokens_neox': 323, 'n_words': 53, 'path': 'scikit-learn/feature_engineering.ipynb', 'repo_name': 'FyzHsn/ml-classification-exploration', 'size': '900'}