<jupyter_start><jupyter_text>Bad-day filter  This notebook is fourth in the series of soiling detection pipeline notebooks (following step after the notebook big_drop_filter.ipynb) Author: <NAME><jupyter_code>import pandas as pd
import numpy as np
import copy
import datetime
from datetime import datetime
import matplotlib.pyplot as plt
from pathlib import Path
from sys import path as syspath
syspath.insert(1, '../src/')
from scripts import apply_filters as af, filter_times as ft<jupyter_output><empty_output><jupyter_text>Import data from previous notebooks<jupyter_code>working_dir = "../data/temp/park1/"
root_path = "../data/temp/park1/big_drop_filter/"
EPI_bigdrop_filt_filepath = root_path + "EPI_bigdrop_filt.csv"
def read_data(EPI_bigdrop_filt_filepath):
    EPI_drops_filt = pd.read_csv(EPI_bigdrop_filt_filepath, delimiter=',',parse_dates=['datetime'], date_parser = pd.to_datetime, index_col='datetime')
    return EPI_drops_filt
EPI_drops_filt = read_data(EPI_bigdrop_filt_filepath)<jupyter_output><empty_output><jupyter_text>Functions<jupyter_code>def bad_day_filter_plot(df, threshold=1): # freqency of df: day
 
    # calculate mean & std of df_string_mean
    anal = pd.Series(df.values.ravel()).describe()
    mean = anal[1]
    std = anal[2]

    # plot histogram of df_string_mean
    ax = df.plot.hist(legend=False,color='orange',alpha=0.1)
    plt.axvline(mean, color='r', linestyle='dashed', linewidth=2)
    plt.axvline(mean - (threshold * std), color='b', linestyle='dashed', linewidth=2)
    plt.axvline(mean + (threshold * std), color='b', linestyle='dashed', linewidth=2)
    plt.figtext(0.1, -0.05, 'mean', horizontalalignment='left', color='r',fontsize=15);
    plt.figtext(0.1, -0.1, 'mean ± threshold * std', horizontalalignment='left', color='b',fontsize=15);
    ax.set_ylabel('Frequency',fontsize=20)
    ax.tick_params(axis = 'both', which = 'major', labelsize = 20)
#     ax.set_xticks(np.arange(0.3, 1.2, 0.2),minor=False)
    ax.set_title('Density plot of mean of each string',fontsize=20)
#     ax.set_xlim(0.25,1.2)<jupyter_output><empty_output><jupyter_text>Apply bad-day filter on big-drop-filtered data<jupyter_code># decide the threshold (std) by plotting histogram: distribution of 'mean of string outputs' 
bad_day_filter_plot(EPI_drops_filt, threshold = 1)
# get boolean for bad (True) and good (False) days
badday_mask = ft.bad_day_filter(EPI_drops_filt, threshold = 2)
# mask (substitute) data with NaN for bad days
EPI_dropsfilt_baddayfilt = EPI_drops_filt.mask(badday_mask)
# Resulting DataFrame: big-drop & bad-day filtered (freq: day)
EPI_dropsfilt_baddayfilt
##save data function from Marcus's scripts
def save_data(dataframes, names, root_dir, sub_dir):

    if root_dir[-1] != "/":
                root_dir += "/"

    if sub_dir[-1] != "/":
                root_dir += sub_dir + "/"

    for data, name in zip(dataframes, names):
        try:
            filepath_out = root_dir + name + ".csv"
            Path(root_dir).mkdir(parents=True, exist_ok=True)
            print(f"\tSaving {filepath_out}...")
            data.to_csv(filepath_out)
            print("\tDone.")
        except Exception as e:
            print(e)
            pass
save_data([EPI_dropsfilt_baddayfilt, badday_mask], ["EPI_dropsfilt_baddayfilt","badday_mask"], working_dir, "bad_day_filter")<jupyter_output>Saving ../data/temp/park1/bad_day_filter/EPI_dropsfilt_baddayfilt.csv...
	Done.
	Saving ../data/temp/park1/bad_day_filter/badday_mask.csv...
	Done.<jupyter_text>Plot: Filtered EPI (big-drop filter & bad-day filter)  Create a Dataframe of 'median EPI of all strings per day'<jupyter_code>EPI_dropsfilt_baddayfilt['median_EPI'] = EPI_dropsfilt_baddayfilt.loc[:, :].median(axis = 1)
EPI_dropsfilt_baddayfilt_daily_median = EPI_dropsfilt_baddayfilt[['median_EPI']]
ymin=0.8
ymax=1.05

fig, ax = plt.subplots(figsize=(8,6))
ax.plot(EPI_dropsfilt_baddayfilt_daily_median.index, EPI_dropsfilt_baddayfilt_daily_median, 'o', markersize = 8,alpha = 0.5, label='median of strings')
ax.set_ylim(ymin,ymax)
fig.autofmt_xdate()
ax.set_yticks(np.arange(ymin,ymax,0.1),minor=False)
ax.set_ylabel('EPI (output/calculated)',fontsize=14)
ax.set_title('Energy Performance Index (EPI)',fontsize=14)
leg = ax.legend();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12903981264637002, 'fraction_numerical': 0.014988290398126464, 'mean_word_length': 5.058156028368795, 'pattern_counts': {'":': 2, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 1, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '597301', 'license': 'MIT', 'n_tokens_mistral': 1649, 'n_tokens_neox': 1566, 'n_words': 304, 'path': 'notebooks/4_bad_day_filter.ipynb', 'repo_name': 'lisc119/Soiling-Loss-Analysis', 'size': '4277'}