<jupyter_start><jupyter_text>End To End Exercise - <NAME><jupyter_code>import warnings
warnings.filterwarnings("ignore")

import numpy as np
import pandas as pd

import matplotlib.pyplot as plt
%matplotlib inline
import seaborn as sns

from datetime import datetime
from sklearn.metrics import mean_squared_error
from math import sqrt 

import statsmodels.api as sm
from statsmodels.tsa.api import Holt
df = pd.read_csv('GlobalLandTemperaturesByCity.csv')
df.head()
def prep_temp_data(df):
    df = df[df.City == 'Xingtai']
    # converts temp to degree farenheit
    df['avg_temp_f'] = (df.AverageTemperature * 9/5) + 32 
    
    return df
df = prep_temp_data(df)
df.head()
df.shape
df.City.value_counts()
df.Country.value_counts()
df.Latitude.value_counts()
df.Longitude.value_counts()
df.AverageTemperature.hist()
df.AverageTemperatureUncertainty.hist()<jupyter_output><empty_output><jupyter_text>Check for null then use ffill or bfill.<jupyter_code>df.isnull().count()
def add_layers(df):
    # Sort rows by the date and then set the index as that date
    df.dt = pd.to_datetime(df.dt)
    df = df.set_index("dt", drop=False).sort_index()
    df['weekday'] = pd.DatetimeIndex(df.dt).day
    df['month'] = pd.DatetimeIndex(df.dt).month
    df['year'] = pd.DatetimeIndex(df.dt).year
    
    # remove leap days
    df = df[df.index != '2016-02-29']
    df = df.fillna(method='ffill')

    return df
df = add_layers(df)
df.head()<jupyter_output><empty_output><jupyter_text>Train, Validate, and Test<jupyter_code>def train_val_test(df):
    train_size = int(len(df) * .5)
    validate_size = int(len(df) * .3)
    test_size = int(len(df) - train_size - validate_size)
    validate_end_index = train_size + validate_size

    train = df[: train_size]
    validate = df[train_size:validate_end_index]
    test = df[validate_end_index:]
    
    check1 = pd.concat([train.tail(1), validate.head(1)])
    check2 = pd.concat([validate.tail(1), test.head(1)])
    
    print(len(train) + len(validate) + len(test) == len(df))
    print(df.head(1) == train.head(1))
    
    return df, train, validate, test, check1, check2
df, train, validate, test, check1, check2 = train_val_test(df)
check1
check2
train.dt.nunique()
df.dt.min()
df.dt.max()
df.dt.max() - df.dt.min()
# 193 years of data (assuming all years are filled)
# China industrial reveluation 1988 (I want to look at 1988 - 2013)
def plot_temp_data(train):
    
    plt.figure(figsize=(20, 30))
    
    plt.subplot(10,1,1)
    train.avg_temp_f.plot()
    
    plt.subplot(10,1,2)
    train.AverageTemperature.plot()
    
    plt.subplot(10,1,3)
    train.AverageTemperatureUncertainty.plot()
plot_temp_data(train)
# Plot mean temperature by month (bar plot)
train.groupby('year').avg_temp_f.mean().plot.bar()
sns.boxplot(data = train, x = 'month', y = 'avg_temp_f')
sns.boxplot(data = train, x = 'year', y = 'avg_temp_f')
train.head()
train.dt = pd.to_datetime(train.dt)
validate.dt = pd.to_datetime(validate.dt)
test.dt = pd.to_datetime(test.dt)

train = train.set_index("dt", drop=False).sort_index()
validate = validate.set_index("dt", drop=False).sort_index()
test = test.set_index("dt", drop=False).sort_index()

train.avg_temp_f.resample('M').mean().plot(label = 'Monthly')
train.avg_temp_f.resample('Y').mean().plot(label = 'Yearly')
plt.legend()<jupyter_output><empty_output><jupyter_text>Evaluate/Model<jupyter_code>def evaluate(target_var):
    rmse = round(sqrt(mean_squared_error(validate[target_var], yhat_df[target_var])), 0)
    return rmse
def plot_and_eval(target_var):
    plt.figure(figsize = (12,4))
    plt.plot(train[target_var], label='Train', linewidth=1)
    plt.plot(validate[target_var], label='Validate', linewidth=1)
    plt.plot(yhat_df[target_var])
    plt.title(target_var)
    rmse = evaluate(target_var)
    print(target_var, '-- RMSE: {:.0f}'.format(rmse))
    plt.show()
# create an empty dataframe
eval_df = pd.DataFrame(columns=['model_type', 'target_var', 'rmse'])

# function to store the rmse so that we can compare
def append_eval_df(model_type, target_var):
    rmse = evaluate(target_var)
    d = {'model_type': [model_type], 'target_var': [target_var],
        'rmse': [rmse]}
    d = pd.DataFrame(d)
    return eval_df.append(d, ignore_index = True)<jupyter_output><empty_output><jupyter_text>Last observed value<jupyter_code>def lov_model(train, validate):
    avg_temp_f = round(train['avg_temp_f'][-1:][0], 2)
    yhat_df = pd.DataFrame({'avg_temp_f': [avg_temp_f]},  
                      index = validate.index)
    return train, validate, avg_temp_f, yhat_df
train, validate, avg_temp_f, yhat_df = lov_model(train, validate)
yhat_df.head()
yhat_df.describe()
plot_and_eval('avg_temp_f')
eval_df = append_eval_df(model_type = 'last_observed_value', 
                             target_var = 'avg_temp_f' )
eval_df<jupyter_output><empty_output><jupyter_text>Simple Average<jupyter_code>def sa_model(train, validate):
    # compute simple average
    # plt.plot(train['dollars_sold'])
    avg_temp_f = round(train['avg_temp_f'].mean(),2)
    avg_temp_f
    
    # make predictions
    yhat_df = pd.DataFrame({'avg_temp_f': [avg_temp_f]}, 
                           index = validate.index)
    
    return train, validate, avg_temp_f, yhat_df
train, validate, avg_temp_f, yhat_df = sa_model(train, validate)
yhat_df.head()
yhat_df.describe()
plot_and_eval('avg_temp_f')
eval_df = append_eval_df(model_type='simple_average', 
                            target_var = 'avg_temp_f')
eval_df<jupyter_output><empty_output><jupyter_text>Moving Average<jupyter_code># train['dollars_sold'].tail(30)
plt.figure(figsize=(12,4))
plt.plot(train['avg_temp_f'].rolling(7).mean())
plt.plot(train['avg_temp_f'].rolling(30).mean())
plt.plot(train['avg_temp_f'].rolling(90).mean())
plt.plot(train['avg_temp_f'].rolling(120).mean())
plt.plot(train['avg_temp_f'], alpha=.3)
periods = [30, 100, 365, 730]

for p in periods:
    avg_temp_f = round(train['avg_temp_f'].rolling(p).mean()[-1], 2)

print(avg_temp_f)
# make predictions
yhat_df = pd.DataFrame({'avg_temp_f': [avg_temp_f]}, 
                           index = validate.index)
yhat_df
yhat_df.head()
plot_and_eval('avg_temp_f')
eval_df = append_eval_df(model_type = '30-730d_moving_avg', 
                            target_var = 'avg_temp_f' )
eval_df
eval_df.rmse.min()
# Best models are simple average and 30-730d moving average with a rmse of 19.
sm.tsa.seasonal_decompose(train.avg_temp_f.resample('M').mean()).plot()
plt.show()
def sa_model_test(test, validate):
    # compute simple average
    # plt.plot(train['dollars_sold'])
    avg_temp_f = round(test['avg_temp_f'].mean(),2)
    avg_temp_f
    
    # make predictions
    yhat_df = pd.DataFrame({'avg_temp_f': [avg_temp_f]}, 
                           index = validate.index)
    
    return test, validate, avg_temp_f, yhat_df
test, validate, avg_temp_f, yhat_df = sa_model_test(test, validate)
yhat_df.head()
yhat_df.describe()
plot_and_eval('avg_temp_f')
eval_df = append_eval_df(model_type='simple_average_test', 
                            target_var = 'avg_temp_f')
eval_df<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14157303370786517, 'fraction_numerical': 0.01544943820224719, 'mean_word_length': 4.374339622641509, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '280555', 'license': 'MIT', 'n_tokens_mistral': 2751, 'n_tokens_neox': 2632, 'n_words': 469, 'path': 'data_modeling/end_to_end_exercise.ipynb', 'repo_name': 'ljackson707/time-series-exercises', 'size': '7126'}