<jupyter_start><jupyter_text>Feature engineering https://www.youtube.com/watch?v=nT6UsVgJ0xw<jupyter_code>import pathlib
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import OneHotEncoder
from sklearn.compose import ColumnTransformer<jupyter_output><empty_output><jupyter_text>Get data<jupyter_code>data_file_name =  pathlib.Path('..') / '..' / 'download_data_dir' / 'air_quality.csv'
data_file_name.exists()
air_df = pd.read_csv(data_file_name, index_col='date_local', parse_dates=True)
air_df.shape<jupyter_output><empty_output><jupyter_text>Prepare data<jupyter_code>air_df['month'] = air_df.index.month
air_df['time'] = np.arange(len(air_df.index))<jupyter_output><empty_output><jupyter_text>Split data into test and train<jupyter_code>air_df_train = air_df.loc[:'2015-4'].copy()
air_df_test = air_df.loc['2015-5':].copy()<jupyter_output><empty_output><jupyter_text>Plot<jupyter_code>fig, ax = plt.subplots(figsize=(18, 6))
air_df_train[['aqi']].plot(ax=ax)
air_df_test[['aqi']].plot(ax=ax)
plt.legend(['train set', 'test set'])
air_df.head()
X_monthly = air_df_train[['time', 'month']]
y_train = air_df_train['aqi'].values
feature_transformer = ColumnTransformer(
    [('categorical', OneHotEncoder(drop='first'), ['month'])],
    remainder='passthrough'
)
model_monthly = Pipeline([
    ('preprocess', feature_transformer),
    ('model', LinearRegression())
])
lm_monthly = model_monthly.fit(X_monthly, y_train)
print(f'R^2 is {round(lm_monthly.score(X_monthly, y_train), 3)}')
air_df['y_pred_monthly'] = lm_monthly.predict(air_df[['time', 'month']])
air_df[['aqi', 'y_pred_monthly']].loc[:'2015-4'].plot(
    figsize=(16, 4), color=['#499DE6', 'red'])
average_season_effect = np.dot(
    lm_monthly['model'].coef_[:-1], [1/12] * 11)
def trend_fn(df):
    return lm_monthly['model'].intercept_ + lm_monthly['model'].coef_[-1] * df['time'] + average_season_effect
simple_ets = (
    air_df[['aqi', 'y_pred_monthly', 'time']].loc[:'2015-4'].rename(
        columns={'aqi': 'y_real',
                 'y_pred_monthly': 'y_hat'}).assign(
    residuals=lambda df: df['y_real'] - df['y_hat'],
    trend=trend_fn,
    seasonal = lambda df: df['y_hat'] - df['trend']
))
simple_ets[['y_real', 'y_hat', 'trend']].plot(figsize=(16, 4))
simple_ets[['seasonal']].plot(figsize=(16,2), c='m')
simple_ets[['residuals']].plot(figsize=(16,2), c='r')
simple_ets = (
    simple_ets.assign(
        deseasonalized=lambda df: df['y_real'] - df['seasonal'],
        detrended=lambda df: df['y_real'] - df['trend']))
simple_ets[['deseasonalized']].plot(figsize=(16, 3), c='g', label='deseasonalized')
simple_ets[['detrended']].plot(figsize=(16, 3), c='g', label='detrended')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1875442321302194, 'fraction_numerical': 0.019815994338287332, 'mean_word_length': 7.24198250728863, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '349986', 'license': 'Apache-2.0', 'n_tokens_mistral': 1154, 'n_tokens_neox': 1116, 'n_words': 154, 'path': 'notebooks/ts-pycon-2021/03-feature-engineering.ipynb', 'repo_name': 'gavinln/ts-analysis', 'size': '2826'}