<jupyter_start><jupyter_text>Pollen level prediction<jupyter_code>#hide
from nbdev.showdoc import *
from nbdev.export import *
notebook2script()
import pandas as pd
import matplotlib.pyplot as plt
from sqlalchemy import create_engine
engine = create_engine('mysql+pymysql://root:PASSWORD@localhost:3306/pollenvarsling-ml')
connection = engine.connect()<jupyter_output><empty_output><jupyter_text>Actual readings<jupyter_code>sql = 'SELECT * FROM `pollenvarsling-ml`.reading_;'
# sql = 'SELECT * FROM `pollenvarsling-ml`.reading_ where type_id=4;'
df = pd.read_sql(sql, connection)
df.to_pickle('pollen.pkl')
df = pd.read_pickle('pollen.pkl')
df
df.plot.hist(y='year_')
df.plot.hist(y='week_')
df.plot.hist(y='day_year')
df.plot.hist(y='value_')
df.plot.hist(y='type_id', bins=45)
df.plot.box(y='value_')
df['value_'].describe()
df.plot.scatter(x='day_year', y='value_')
def plot_scatter_for_type(df, type):
    year_start = 2016
    year_end = 2020
    for year in range(year_start, year_end):
        df[df.year_ == year][df.type_id == type].plot.scatter(y='value_', x='day_year', xlim=(0,365), c=['blue', 'red', 'green', 'brown'][year - year_start])  
        plt.title(f'Type {type}, {year}')
    plt.show()
    
for type in range(1, 3):
    plot_scatter_for_type(df, type)<jupyter_output>/home/claus/anaconda3/envs/nbdev_exploration/lib/python3.7/site-packages/ipykernel_launcher.py:5: UserWarning: Boolean Series key will be reindexed to match DataFrame index.
  """<jupyter_text>Remove unused columns<jupyter_code>pollen_type = 0
df.drop(df[df.type_id != pollen_type].index, inplace=True)
drop_cols = ['id_', 'created_', 'created_by', 'offset_', 'at_', 'week_', 'week_year', 'day_week', 'day_month', 'corrected_', 'factor_', 'status_id', 'type_id']
df.drop(drop_cols, axis=1, inplace=True)
df.dropna(inplace=True)
df = df.rename(columns={'year_': 'year'})
df
df.to_pickle('pollen_0.pkl')
df = pd.read_pickle('pollen_0.pkl')<jupyter_output><empty_output><jupyter_text>Weather forecast<jupyter_code>sql = 'SELECT * FROM `pollenvarsling-ml`.forecast;'
df = pd.read_sql(sql, connection)
df.describe()<jupyter_output><empty_output><jupyter_text>Weather, actual<jupyter_code>sql = 'SELECT * FROM `pollenvarsling-ml`.weather_cph_20190404;'
df = pd.read_sql(sql, connection)
df.to_pickle('weather.pkl')
df = pd.read_pickle('weather.pkl')
df.describe()
df<jupyter_output><empty_output><jupyter_text>Convert date to day of year<jupyter_code>import datetime
df['date_str'] = df['År'].astype(str) + "/" + df['Måned'].astype(str) + "/" + df['Dag'].astype(str)
df['day_year'] = pd.to_datetime(df['date_str'].astype(str)).dt.dayofyear<jupyter_output><empty_output><jupyter_text>Remove unused columns<jupyter_code>drop_cols = ['Station', 'Navn', 'date_str', 'Måned', 'Dag', 'Middel relativ fugtighed (%)', 'Højeste relativ fugtighed (%)', 'Laveste relativ fugtighed (%)']
df.drop(drop_cols, axis=1, inplace=True)
df.dropna(inplace=True)<jupyter_output><empty_output><jupyter_text>Rename columns<jupyter_code>df = df.rename(columns={'År': 'year'})
df
df.describe()
df.to_pickle('weather_0.pkl')
df = pd.read_pickle('weather_0.pkl')<jupyter_output><empty_output><jupyter_text>Merge pollen and actual weather<jupyter_code>weather = pd.read_pickle('weather_0.pkl')
pollen = pd.read_pickle('pollen_0.pkl')
df = weather.append(pollen, ignore_index=True, sort=True)
df
df.describe()
from numpy import nanmean

from functools import partial
s_na_mean = partial(pd.Series.mean, skipna = True)

aggregate = {
    'Akkumuleret nedbør (mm)': [nanmean],
    'Maksimumtemperatur (grader C)': [nanmean],
    'Middellufttryk (hPa)': [nanmean],
    'Middeltemperatur (grader C)': [nanmean],
    'Middelvindhastighed (m/s)': [nanmean],
    'Middelvindretning (grader)': [nanmean],
    'Minimumtemperatur (grader C)': [nanmean],
    'hour_': [nanmean],
    'value_': [nanmean]
}
df = df.groupby(['year', 'day_year'], as_index=False).agg(aggregate)
df.describe()
df
df[df['year'] == 2019][df['day_year'] == 215]
from fastai.tabular import * 
dep_var = ''<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1721756290083868, 'fraction_numerical': 0.0123334977799704, 'mean_word_length': 7.093812375249501, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '274077', 'license': 'Apache-2.0', 'n_tokens_mistral': 1668, 'n_tokens_neox': 1609, 'n_words': 258, 'path': '00_core.ipynb', 'repo_name': 'claust/nbdev_exploration', 'size': '4054'}