<jupyter_start><jupyter_text><jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
from matplotlib.dates import DateFormatter, WeekdayLocator
from datetime import timedelta
titles = ['new_cases_per_million', 'total_deaths_per_million', 'total_vaccinations_per_hundred']
df = pd.read_csv(
    'https://covid.ourworldindata.org/data/owid-covid-data.csv', 
    usecols=['date', 'location', 'total_vaccinations_per_hundred'], 
    parse_dates=['date'])

countries = ['United States', 'Germany', 'United Kingdom', 'Israel', 'Japan', 'Brazil', 'India', 'Mongolia']
df = df[df['location'].isin(countries)]
pivot = pd.pivot_table(
    data=df,                                    # What dataframe to use
    index='date',                               # The "rows" of your dataframe
    columns='location',                         # What values to show as columns
    values=titles[-1],    # What values to aggregate
    aggfunc='mean',                             # How to aggregate data
    )

pivot = pivot.fillna(method='ffill')
pivot.head()
import matplotlib.colors as pltc
import random
color = [k for k,v in pltc.cnames.items()]
main_country = 'Japan'
colors = {country:(color.pop(random.randint(0,len(color)-1)) if country!= main_country else '#129583') for country in countries}
print(colors)

# print(colors)
alphas = {country:(0.7 if country!= main_country else 1.0) for country in countries}
fig, ax = plt.subplots(figsize=(12,8))
fig.patch.set_facecolor('#F5F5F5')    # Change background color to a light grey
ax.patch.set_facecolor('#000000')     # Change background color to a light grey
# for title in titles:
for country in countries:
    ax.plot(
        pivot.index,              # What to use as your x-values
        pivot[country],           # What to use as your y-values
        color=colors[country],    # How to color your line
        alpha=alphas[country]     # What transparency to use for your line
    )
    ax.set_yscale('log')
    ax.text(
        x = pivot.index[-1] + timedelta(days=2),    # Where to position your text relative to the x-axis
        y = pivot[country].max(),                   # How high to position your text
        color = colors[country],                    # What color to give your text
        s = country,                                # What to write
        alpha=alphas[country]                       # What transparency to use
    )
date_form = DateFormatter("%Y-%m-%d")
ax.xaxis.set_major_locator(WeekdayLocator(byweekday=(0), interval=1))
ax.xaxis.set_major_formatter(date_form)
plt.xticks(rotation=45)
# plt.ylim(0,100)

## B) Customizing axes and adding a grid
ax.spines['top'].set_visible(False)
ax.spines['right'].set_visible(False)
ax.spines['bottom'].set_color('#3f3f3f')
ax.spines['left'].set_color('#3f3f3f')
ax.tick_params(colors='#3f3f3f')
ax.grid(alpha=0.1)

## C) Adding a title and axis labels
# plt.ylabel(title, fontsize=12, alpha=0.9)
plt.xlabel('Date', fontsize=12, alpha=0.9)
plt.title(titles[-1], fontsize=18, weight='bold', alpha=0.9)

# D) Celebrate!
plt.show()
!pip install catboost
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sklearn.metrics as metrics

import xgboost as xgb
from catboost import CatBoostRegressor
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Lasso
from sklearn.linear_model import Ridge
from sklearn.linear_model import ElasticNet
from sklearn.linear_model import HuberRegressor
from sklearn.linear_model import LassoLars
from sklearn.linear_model import PassiveAggressiveRegressor
from sklearn.linear_model import SGDRegressor

from sklearn.ensemble import AdaBoostRegressor
from sklearn.ensemble import BaggingRegressor
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import RandomForestRegressor
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.tree import DecisionTreeRegressor
from sklearn.tree import ExtraTreeRegressor

def set_date_features(df):
    df['month'] = pd.DatetimeIndex(df.index).month
    df['dayofweek'] = pd.DatetimeIndex(df.index).dayofweek
    return df

# one-hot encoding
def encoding(df):
    df = pd.get_dummies(df, columns=['month'], prefix='month')
    df = pd.get_dummies(df, columns=['dayofweek'], prefix='dayofweek')
    return df

def plot_result(train_y, test_y, y_hat, model_name, df, max):
  data = np.concatenate([train_y, test_y])
  data = data*max
  y_hat = y_hat*max
  plt.plot(df.index, data, label='Original')
  plt.plot(df.index[-len(y_hat):],y_hat, label=model_name)
  plt.legend()
  plt.grid()
  plt.show()

# prepare a dictionary of ml models
def get_models(models=dict()):
    # linear models
    normalize=False
    max_iter=5000
    models['lr'] = LinearRegression()
    models['lasso'] = Lasso()
    models['ridge'] = Ridge()
    models['en'] = ElasticNet()
    models['llars'] = LassoLars()
    # non-linear models
    models['knn'] = KNeighborsRegressor(n_neighbors=7)
    models['cart'] = DecisionTreeRegressor()
    models['extra'] = ExtraTreeRegressor()
    # # ensemble models
    n_trees = 10000
    models['bag'] = BaggingRegressor(n_estimators=n_trees)
    models['rf'] = RandomForestRegressor(n_estimators=n_trees, max_depth=10, random_state=10)
    models['et'] = ExtraTreesRegressor(n_estimators=n_trees)
    # one by one target
    models['huber'] = HuberRegressor()
    models['pa'] = PassiveAggressiveRegressor(tol=1e-3)
    models['sgd'] = SGDRegressor(tol=1e-3)
    models['svmr'] = SVR()
    models['ada'] = AdaBoostRegressor(n_estimators=n_trees)
    models['gbm'] = GradientBoostingRegressor(n_estimators=n_trees)
    models['xgb'] = xgb.XGBRegressor(tree_method='gpu_hist', gpu_id=0, n_estimators=n_trees, max_depth=6, n_jobs=-1, seed=10)
    models['cst'] = CatBoostRegressor(iterations=max_iter, task_type="GPU", devices='0', verbose=False)
    print('Defined %d models' % len(models))

    return models

def regression_results(y_true, y_pred):    # Regression metrics
    explained_variance=metrics.explained_variance_score(y_true, y_pred)
    mean_absolute_error=metrics.mean_absolute_error(y_true, y_pred) 
    mse=metrics.mean_squared_error(y_true, y_pred) 
    # mean_squared_log_error=metrics.mean_squared_log_error(y_true, y_pred)
    median_absolute_error=metrics.median_absolute_error(y_true, y_pred)
    r2=metrics.r2_score(y_true, y_pred)    
    print('explained_variance: ', round(explained_variance,4))    
    # print('mean_squared_log_error: ', round(mean_squared_log_error,4))
    print('r2: ', round(r2,4))
    print('MAE: ', round(mean_absolute_error,4))
    print('MSE: ', round(mse,4))
    print('RMSE: ', round(np.sqrt(mse),4))
split_ratio = 0.9                               # Training and Test data ratio ex: 0.9 -> 90% train 10% test
countries = ['Japan'] #['United States', 'Germany', 'United Kingdom', 'Israel', 'Japan', 'Brazil', 'India', 'Mongolia']
prediction = ['new_cases_per_million', 'total_deaths_per_million', 'total_vaccinations_per_hundred']
index = 0                                       # Which feature to "Predict"
print('predicting ', prediction[index])



models = get_models()                           # Loading models
df = pd.read_csv(                               # Loading data
      'https://covid.ourworldindata.org/data/owid-covid-data.csv', 
      usecols=['date', 'location', 'new_cases_per_million', 'total_deaths_per_million', 'total_vaccinations_per_hundred'],     # Change data: copy and paste following list
      parse_dates=['date'])                     # ['new_cases_per_million', 'total_deaths_per_million', 'total_vaccinations_per_hundred']
df = df[df['location'].isin(countries)]
pivot = pd.pivot_table(
    data=df,                                    # What dataframe to use
    index='date',                               # The "rows" of your dataframe
    columns='location',                         # What values to show as columns
    values=['new_cases_per_million', 'total_deaths_per_million', 'total_vaccinations_per_hundred'],    # What values to aggregate
    aggfunc='mean',                             # How to aggregate data
    )

pivot = pivot.fillna(method='ffill')            # Preprocessing data
pivot = pivot.dropna()
pivot = set_date_features(pivot)
print(pivot.head())
# pivot = encoding(pivot)
df = pivot
pivot = pivot.to_numpy()

max = np.max(pivot[:,index])                        # Splitting data to train and test set
print('Maximum value: ',max)
train_X = pivot[:int(len(pivot)*split_ratio)]
test_X = pivot[int(len(pivot)*split_ratio):]
train_y = pivot[:int(len(pivot)*split_ratio), index]/max
test_y = pivot[int(len(pivot)*split_ratio):, index]/max
print('Normalized value: ', train_y, test_y)

print('Training and test data shapes: ', train_X.shape, train_y.shape, test_X.shape, test_y.shape)
scores_valid = {}
scores_test = {}
print('Training models ... ')                   # Training models
for idx, model_name in enumerate(models):
  print('\n===================================================================================\n')
  model = models[model_name]
  print(model_name, model)
  model.fit(train_X, train_y)
  y_hat = model.predict(test_X)
  scores_valid[model_name] = np.sqrt(mean_squared_error(test_y,y_hat))
  print('Model %i %s MSE On validation Data: '%(idx, model_name), mean_squared_error(test_y,y_hat))
  regression_results(test_y, y_hat)
  plot_result(train_y, test_y, y_hat, model_name, df, max)
print(scores_valid)<jupyter_output>predicting  new_cases_per_million
Defined 19 models
           new_cases_per_million total_deaths_per_million  ... month dayofweek
location                   Japan                    Japan  ...                
date                                                       ...                
2021-02-17                11.552                   57.070  ...     2         2
2021-02-18                12.058                   57.671  ...     2         3
2021-02-19                10.255                   58.193  ...     2         4
2021-02-20                 9.757                   58.809  ...     2         5
2021-02-21                 8.191                   59.181  ...     2         6

[5 rows x 5 columns]
Maximum value:  62.573
Normalized value:  [0.18461637 0.19270292 0.16388858 0.15592987 0.13090311 0.09312323
 0.13937321 0.11423457 0.13697601 0.13355601 0.15415595 0.12598085
 0.08693846 0.11650392 0.1571924  0.14519042 0.14266537 0.05206719
 0.21758586 0.07846835 0.14846659 0.1590782  0.1646[...]
{'fraction_non_alphanumeric': 0.12504725897920604, 'fraction_numerical': 0.038752362948960305, 'mean_word_length': 3.2511048613901163, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '657644', 'license': 'MIT', 'n_tokens_mistral': 3808, 'n_tokens_neox': 3413, 'n_words': 753, 'path': 'vaccination.ipynb', 'repo_name': 'nani0819/dictionary', 'size': '10580'}