<jupyter_start><jupyter_text>Exploratory Data Analysis<jupyter_code>import datetime
import calendar
import json
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from matplotlib import rcParams
rcParams['figure.figsize'] = 14, 8<jupyter_output><empty_output><jupyter_text>Load project dataAfter loading we will have series:  * Flow Sampled at 5 minutes interval  * Rainfall sampled at 1 hour interval  * No NaNs  * Starting and ending at the same time<jupyter_code>project_folder = '../../datasets/radon-medium/'
df = pd.read_csv(project_folder + 'flow1.csv', parse_dates=['time'])
flow = df.set_index('time')['flow'].fillna(0).resample('5T').pad()
df = pd.read_csv(project_folder + 'rainfall1.csv', parse_dates=['time'])
rainfall = df.set_index('time')['rainfall'].fillna(0).resample('5T').sum()
print('Flow date range    : {:} - {:}'.format(flow.index[0], flow.index[-1]))
print('Rainfall date range: {:} - {:}'.format(rainfall.index[0], rainfall.index[-1]))<jupyter_output>Flow date range    : 2013-09-09 10:50:00 - 2017-11-10 14:40:00
Rainfall date range: 2014-10-24 00:00:00 - 2017-11-10 19:25:00<jupyter_text>Based on the data we will use the following range: **2014-10-24** - **2017-11-09**<jupyter_code>flow = flow['2014-10-24':'2017-11-09']
rainfall = rainfall['2014-10-24':'2017-11-09']
print('Flow date range    : {:} - {:}'.format(flow.index[0], flow.index[-1]))
print('Rainfall date range: {:} - {:}'.format(rainfall.index[0], rainfall.index[-1]))<jupyter_output>Flow date range    : 2014-10-24 00:00:00 - 2017-11-09 23:55:00
Rainfall date range: 2014-10-24 00:00:00 - 2017-11-09 23:55:00<jupyter_text>Helper functions<jupyter_code>df = flow.to_frame().reset_index()
daily_pattern = df.groupby(by=[df.time.map(lambda x: x.time())]).flow.median()


def plot_days(xs, start_day, end_day):
    """Plot series for specific data range"""
    df = xs[pd.Timestamp(start_day): pd.Timestamp(end_day)]
    df.plot()
    plt.show()    


def plot_rdii(start_day, end_day):
    xs = flow[pd.Timestamp(start_day): pd.Timestamp(end_day)]
    base_flow = pd.Series(xs.index.map(lambda x: daily_pattern[x.time()]), xs.index)
    ys = rainfall[pd.Timestamp(start_day): pd.Timestamp(end_day)]
    # Plot rainfall
    plt.figure(1)
    plt.subplot(211)
    ax = ys.plot.bar(width=0.1, edgecolor='lightblue', color='lightblue', ylim=(5, 0))
    ax.set_xticks([])
    # Plot flow
    plt.subplot(212)
    xs.plot()
    base_flow.plot()
    plt.show()<jupyter_output><empty_output><jupyter_text>Find biggest stormsTo find biggest storms we will sum all rainfalls for each day, and the select days with the biggest total precipitation<jupyter_code>df = rainfall.resample('D').sum()
df.sort_values(ascending=False).head(10)<jupyter_output><empty_output><jupyter_text>Plot storm days<jupyter_code>plot_rdii('2015-10-27', '2015-10-31')
plot_rdii('2017-06-15', '2017-06-25')
plot_rdii('2017-06-17 13:00', '2017-06-17 18:00')
plot_rdii('2015-08-09', '2015-08-12')
plot_rdii('2017-05-20', '2017-06-02')
plot_rdii('2017-04-01', '2017-05-01')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17360208062418725, 'fraction_numerical': 0.09622886866059818, 'mean_word_length': 5.318275154004107, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '518643', 'license': 'Apache-2.0', 'n_tokens_mistral': 1359, 'n_tokens_neox': 1138, 'n_words': 219, 'path': 'notebooks/radon-medium/[EDA] storms.ipynb', 'repo_name': 'carldata/argo', 'size': '3076'}