<jupyter_start><jupyter_text>Big drop filter  This notebook is third in the series of soiling detection pipeline notebooksData from other parks can be used in the following notebooks by changing the working_dir and filepaths     CSVs imported in this notebook are created in the 2_detect_timewindow pipeline notebook     Author: <NAME><jupyter_code>import pandas as pd
import numpy as np
import matplotlib
import copy
import matplotlib.pyplot as plt
from pathlib import Path

import datetime
from datetime import datetime<jupyter_output><empty_output><jupyter_text>Import dataframes from previous notebook<jupyter_code>working_dir = "../data/temp/park2/"

root_path = "../data/temp/park2/time_mask_filter/"
EPI_stringfilt_path = root_path + "EPI_stringfilt.csv"
EPI_timemask_path = root_path + "EPI_timemask.csv"
def read_data(EPI_stringfilt_path, EPI_timemask_path):
    EPI_stringfilt = pd.read_csv(EPI_stringfilt_path, delimiter=',',parse_dates=['datetime'], date_parser = pd.to_datetime, index_col='datetime')
    EPI_timemask = pd.read_csv(EPI_timemask_path, delimiter = ',',parse_dates=['datetime'], date_parser = pd.to_datetime, index_col='datetime')
    return [EPI_stringfilt, EPI_timemask]
EPI_stringfilt, EPI_timemask = read_data(EPI_stringfilt_path, EPI_timemask_path)<jupyter_output><empty_output><jupyter_text>Filter by the change in EPI from one day to the next  Aggregate by day and calculate the daily change in EPI<jupyter_code>EPI_timemask_D= EPI_timemask.resample(rule='D').mean()
lagdf_D = pd.DataFrame(copy.deepcopy(EPI_timemask_D))
for cols in lagdf_D.columns:
    lagdf_D[cols]=lagdf_D[cols]- lagdf_D[cols].shift(1)
    

leaddf_D = pd.DataFrame(copy.deepcopy(lagdf_D))
for cols in leaddf_D.columns:
    leaddf_D[cols]= leaddf_D[cols]-leaddf_D[cols].shift(-1)<jupyter_output><empty_output><jupyter_text>Set thresholds for large drops/jumps<jupyter_code>description_dailyagg = lagdf_D.describe()
mean_lag= description_dailyagg.loc['mean'].mean()
std_lag= description_dailyagg.loc['std'].mean()

lower_thresh_lag = mean_lag-std_lag
upper_thresh_lag = -lower_thresh_lag
lower_thresh_lag
description_dailyagg = leaddf_D.describe()
mean_lead= description_dailyagg.loc['mean'].mean()
std_lead= description_dailyagg.loc['std'].mean()

lower_thresh_lead = mean_lag-std_lag
upper_thresh_lead = -lower_thresh_lag
lower_thresh_lead<jupyter_output><empty_output><jupyter_text>Create boolean masks for removing big drops/jumpsAlso generates a mask for only removing big drops (drop_mask), if wanted<jupyter_code>bool_1=lagdf_D.lt(lower_thresh_lag)
bool_2 =lagdf_D.gt(upper_thresh_lag)

bool_3=leaddf_D.lt(lower_thresh_lead)
bool_4 =leaddf_D.gt(upper_thresh_lead)


#if only using the drop filter
drop_mask=bool_1|bool_2
#if use the drop and jump filter: stringmask2
drop_jump_mask=bool_1|bool_2|bool_3|bool_4



EPI_bigdrop_filt = EPI_timemask_D.mask(drop_jump_mask)<jupyter_output><empty_output><jupyter_text>Visualise to check effect of filter<jupyter_code>EPI_timemask_D.median(axis=1).plot()
EPI_bigdrop_filt.median(axis=1).plot()<jupyter_output><empty_output><jupyter_text>Save big-drop filtered EPIs, and the masks for filtering<jupyter_code>##save data function from Marcus's scripts
def save_data(dataframes, names, root_dir, sub_dir):

    if root_dir[-1] != "/":
                root_dir += "/"

    if sub_dir[-1] != "/":
                root_dir += sub_dir + "/"

    for data, name in zip(dataframes, names):
        try:
            filepath_out = root_dir + name + ".csv"
            Path(root_dir).mkdir(parents=True, exist_ok=True)
            print(f"\tSaving {filepath_out}...")
            data.to_csv(filepath_out)
            print("\tDone.")
        except Exception as e:
            print(e)
            pass

save_data([EPI_bigdrop_filt,drop_jump_mask,drop_mask], ["EPI_bigdrop_filt",'drop_jump_mask','drop_mask'], working_dir, "big_drop_filter")<jupyter_output>Saving ../data/temp/park2/big_drop_filter/EPI_bigdrop_filt.csv...
	Done.
	Saving ../data/temp/park2/big_drop_filter/drop_jump_mask.csv...
	Done.
	Saving ../data/temp/park2/big_drop_filter/drop_mask.csv...
	Done.
{'fraction_non_alphanumeric': 0.10583941605839416, 'fraction_numerical': 0.00559610705596107, 'mean_word_length': 5.66288492706645, 'pattern_counts': {'":': 2, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '574521', 'license': 'MIT', 'n_tokens_mistral': 1537, 'n_tokens_neox': 1487, 'n_words': 275, 'path': 'notebooks/3_big_drop_filter.ipynb', 'repo_name': 'lisc119/Soiling-Loss-Analysis', 'size': '4117'}