<jupyter_start><jupyter_text>Introduction<jupyter_code># In this file, we will divide the counties in US into 4 groups according to their reported deaths number.
# Then we will train different models for different groups of counties using curve fitting and svm.

# The estimated running time of this jupyter notebook should be less than 15 minutes.<jupyter_output><empty_output><jupyter_text>Dependencies<jupyter_code>from linear_regression import LinearRegressor
from svm import SVM
from output import Output
from model_scoring_evaluation import generate_day_tag, score_all_predictions
from preprocessForNN import PreprocessForNN
import pandas as pd
import numpy as np<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code># We should transform the data of New York Times into daily deaths and confirmed cases.
# In fact, the preprocess has been done and the processed data is saved as a .csv file.
# There is no need to run this section of code. So we can skip this step to save some time.
preprocess = PreprocessForNN()
preprocess.transform_format('death')
preprocess.transform_format('confirmed')<jupyter_output><empty_output><jupyter_text>Linear Regression<jupyter_code># First of all, we fit the death data from the counties that reported less than 14 deaths during the past two weeks.
lr = LinearRegressor()
lr.train()
lr.test()
# Then we can write our predictions to 'sample_submission.csv' in order to calculate pinball loss.
source = 'models/LR/lr_burning.csv'
dst = 'submissions/submission_svm_lr_model.csv'
output = Output()
output.save_submission(source, dst)
# Secondly, we fit the death data from the counties that reported less than 140 deaths during the past two weeks.
lr = LinearRegressor()
lr.train('mid')
lr.test('mid')
# Again, we write our submission to our output file.
source = 'models/LR/lr_mid.csv'
dst = 'submissions/submission_svm_lr_model.csv'
output = Output('submissions/submission_svm_lr_model.csv')
output.save_submission(source, dst)<jupyter_output>0/293293
100000/293293
200000/293293
Successfully saved!<jupyter_text>SVM<jupyter_code># Then we use SVM model to fit counties with more deaths. We divide these counties into two groups and we will build different models for them.
# So we fit the data of counties which reported more than 140 but less than 700 deaths during past two weeks using SVM model.
s = SVM()
s.train('mid2')
s.test('mid2')
source = 'models/SVM/svm_mid2.csv'
dst = 'submissions/submission_svm_lr_model.csv'
output = Output('submissions/submission_svm_lr_model.csv')
output.save_submission(source, dst)
# Finally, we fit the data of counties which reported more than 700 deaths during past two weeks.
s = SVM()
s.train()
s.test()
source = 'models/SVM/svm_outbreak.csv'
dst = 'submissions/submission_svm_lr_model.csv'
output = Output('submissions/submission_svm_lr_model.csv')
output.save_submission(source, dst)<jupyter_output>0/293293
100000/293293
200000/293293
Successfully saved!<jupyter_text>Pinball Loss<jupyter_code># Now we can calculate pinball loss of our model to see the performance.
pred_file = 'submissions/submission_svm_lr_model.csv'

start_date = '2020-05-18'
predicted_length = 7
date_list = generate_day_tag(start_date, predicted_length)
for day in date_list:
    scores = score_all_predictions(pred_file, day, '2020-05-17', key='deaths')
    scores_mse = score_all_predictions(pred_file, day, '2020-05-17', key='deaths', mse=True)
    print("Day %s: pinball=%f mse=%f" % (day, scores[0], scores_mse[0]))<jupyter_output>Day 2020-05-18: pinball=0.141824 mse=5.507602
Day 2020-05-19: pinball=0.229476 mse=9.346572
Day 2020-05-20: pinball=0.229755 mse=8.532733
Day 2020-05-21: pinball=0.206330 mse=5.901334
Day 2020-05-22: pinball=0.198107 mse=7.022960
Day 2020-05-23: pinball=0.162344 mse=4.504189
Day 2020-05-24: pinball=0.092802 mse=3.150791
{'fraction_non_alphanumeric': 0.09026871901904514, 'fraction_numerical': 0.06861466214453431, 'mean_word_length': 6.07380073800738, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '550895', 'license': 'MIT', 'n_tokens_mistral': 1465, 'n_tokens_neox': 1265, 'n_words': 395, 'path': 'OwO_29_code_demo.ipynb', 'repo_name': 'Zhouzhiling/OwO_covid', 'size': '3833'}