<jupyter_start><jupyter_text>Import data<jupyter_code>rootPath = 'D:/Dropbox/Project/code/Python/COVID19-Global-Forecasting/data/'
trainFileName = 'train.csv'
testFileName = 'test.csv'
trainWithFlightFileName = 'train_flight.csv'
enrichedDataFileName = 'enriched_covid_19_week_2.csv'<jupyter_output><empty_output><jupyter_text>Load file<jupyter_code>def loadDataByCountry(Country, data_all=None, dateRange= ('2020-01-22','2020-04-13'), is_startFromFirstCaseDay=False, daily=False):
    confirmed_total_date = data_all[data_all['Country_Region']==Country].groupby(['Date']).agg({'ConfirmedCases':['sum']})
    flight_total_date = data_all[data_all['Country_Region']==Country].groupby(['Date']).agg({'DepartureFlight':['mean']})
    data = data_all[data_all['Country_Region']==Country].groupby(['Date']).agg({'ConfirmedCases':['sum'],
                                                                                'Fatalities':['sum'],
                                                                                'DepartureFlight':['mean'],
                                                                                'hospibed':['mean'],
                                                                                'lung':['mean'],
                                                                                'total_pop':['mean'],
                                                                                'density':['mean'],
                                                                                'age_100+':['mean']})
    day = flight_total_date.shape[0]
#     for i in tqdm(range(day), desc='Complete zero-value in flight data'):
#         if flight_total_date.iloc[i].values[0]==0:
#             flight_total_date.iloc[i] = int((flight_total_date.iloc[i-1]+flight_total_date.iloc[i+1])/2)
    
    is_FirstCase = False
    FirstCaseDate = dateRange[0]
    for i in tqdm(range(day), desc='Complete zero-value in data'):
        Date = data.iloc[i].name
        if not is_FirstCase:
            if data.iloc[i].values[1]!=0:
                print('For ', Country, 'The first case during the period just has been found', Date)
                FirstCaseDate = Date
                is_FirstCase = True
            
        if data.iloc[i].values[2]==0:
            data.loc[Date,'DepartureFlight'] = int((data.iloc[i-1]['DepartureFlight']+data.iloc[i+1]['DepartureFlight'])/2)
    
    if daily:
        data_ConfirmedCases = data.ConfirmedCases.diff()
        data_Fatalities = data.Fatalities.diff()
        
        if not is_startFromFirstCaseDay:
            data.ConfirmedCases = data_ConfirmedCases
            data.Fatalities = data_Fatalities
            return data.iloc[1:].loc[dateRange[0]: dateRange[1]]
        else:
            data.ConfirmedCases = data_ConfirmedCases
            data.Fatalities = data_Fatalities
            return data.iloc[1:].loc[FirstCaseDate: dateRange[1]]
    
    if not is_startFromFirstCaseDay:   
        return data.loc[dateRange[0]: dateRange[1]]
    else:
        return data.loc[FirstCaseDate: dateRange[1]]

# dateRange= ('2020-01-22','2020-04-13')
# data.iloc[1:].loc[dateRange[0]: dateRange[1]]
# loadDataByCountry(Country = 'China', data_all=rich_data, is_startFromFirstCaseDay=True, daily=True)
def load_extra_data(extra_data_df=None):
    extra_data_df["Country_Region"] = [country_name.replace("'","") if country_name.find('_')<0 else country_name[:country_name.find('_')].replace("'","") for country_name in extra_data_df["Country_Region"]]

#     extra_data_df["restrictions"] = extra_data_df["restrictions"].astype("int")
#     extra_data_df["quarantine"] = extra_data_df["quarantine"].astype("int")
#     extra_data_df["schools"] = extra_data_df["schools"].astype("int")
    extra_data_df["total_pop"] = extra_data_df["total_pop"].astype("float")
    extra_data_df["density"] = extra_data_df["density"].astype("float")
    extra_data_df["hospibed"] = extra_data_df["hospibed"].astype("float")
    extra_data_df["lung"] = extra_data_df["lung"].astype("float")
    extra_data_df["age_100+"] = extra_data_df["age_100+"].astype("float")
    
    extra_data_df["total_pop"] = extra_data_df["total_pop"]/max(extra_data_df["total_pop"])
    extra_data_df["density"] = extra_data_df["density"]/max(extra_data_df["density"])
    extra_data_df["hospibed"] = extra_data_df["hospibed"]/max(extra_data_df["hospibed"])
    extra_data_df["lung"] = extra_data_df["lung"]/max(extra_data_df["lung"])
    extra_data_df["age_100+"] = extra_data_df["age_100+"]/max(extra_data_df["age_100+"])

    extra_data_df = extra_data_df[["Country_Region","Date","hospibed","lung","total_pop","density","age_100+"]]
    return extra_data_df
def merge_extra_data(extra_data_df=None, raw_data=None):
    extra_data = load_extra_data(extra_data_df=extra_data_df)
    raw_data["Country_Region"] = [country_name.replace("'","") for country_name in raw_data["Country_Region"]]
    train_df = raw_data.merge(extra_data, how="left", on=['Country_Region','Date']).drop_duplicates()
    
    for country_region in train_df.Country_Region.unique():
        query_df = train_df.query("Country_Region=='"+country_region+"' and Date=='2020-03-25'")
        train_df.loc[(train_df["Country_Region"]==country_region) & (train_df["Date"]>"2020-03-25"),"total_pop"] = query_df.total_pop.values[0]
        train_df.loc[(train_df["Country_Region"]==country_region) & (train_df["Date"]>"2020-03-25"),"hospibed"] = query_df.hospibed.values[0]
        train_df.loc[(train_df["Country_Region"]==country_region) & (train_df["Date"]>"2020-03-25"),"density"] = query_df.density.values[0]
        train_df.loc[(train_df["Country_Region"]==country_region) & (train_df["Date"]>"2020-03-25"),"lung"] = query_df.lung.values[0]
        train_df.loc[(train_df["Country_Region"]==country_region) & (train_df["Date"]>"2020-03-25"),"age_100+"] = query_df["age_100+"].values[0]
    return train_df<jupyter_output><empty_output><jupyter_text>LSTM model<jupyter_code>def root_mean_squared_error(y_true, y_pred):
    return K.sqrt(K.mean(K.square(K.log(y_pred) - K.log(y_true)), axis=-1)) 
def aLSTM(time_step = 6, nfeatures=3):
    inputs = Input(shape=(time_step,nfeatures))
    lstm1 = LSTM(128, return_sequences = True, recurrent_dropout=0.5)(inputs)
    
    # Confiromed cases
    lstmC = LSTM(64)(lstm1)
    denseC1 = Dense(32, activation='relu')(lstmC)
    dropC1 = Dropout(0.5)(denseC1)
    denseC2 = Dense(2, activation='relu')(dropC1)
    
    # Fatalities
#     lstmF = LSTM(32)(lstm1)
#     denseF1 = Dense(128)(lstmF)
#     dropF1 = Dropout(0.5)(denseF1)
#     denseF2 = Dense(1, activation='relu')(dropF1)
    
    model = Model(inputs=inputs, outputs=denseC2)
#     
    model.compile(loss=MeanSquaredLogarithmicError(), optimizer='adam', metrics = ['accuracy'])
    model.summary()
    return model
    
# rich_data.Country_Region.unique()<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>def multi_country(country, data_all, is_startFromFirstCaseDay, daily, length):
    data = loadDataByCountry(Country = country, data_all=rich_data, is_startFromFirstCaseDay=True, daily=daily)
    data_CandF = data.values[:,0:2]
    # Consider all features
    data_multi_features = data.values
    scaled_data_b,scb = MaxMinNormalization2(data = data_CandF)
    scaled_data_c, scc = MaxMinNormalization2(data = data_multi_features)
    data_gen = TimeseriesGenerator(scaled_data_c,scaled_data_b, length =length , batch_size=1)
    X_data = []
    Y_data = []
    for i in range(len(data_gen)):
        X,Y= data_gen[i]
        X_data.append(X[0])
        Y_data.append(Y[0])
    return X_data, Y_data
data_test = pd.read_csv(rootPath+testFileName)
data_all = pd.read_csv(rootPath+trainWithFlightFileName)
data_all.Province_State.fillna("None", inplace=True)
# data_all = data_all.drop(['Province_State'], axis=1)
extra_data_df = pd.read_csv(rootPath+enrichedDataFileName)
rich_data = merge_extra_data(extra_data_df=extra_data_df, raw_data=data_all)
dayList =datelist ('20200122','20200413')
split = 0.6
save_dir = "saved_models"
save_fname = os.path.join(save_dir, 'COVID19_LSTM.h5')
time_step = 6
daily=True

X_data = []
Y_data = []

is_single_country = False
country = 'China'
if is_single_country:
    data = loadDataByCountry(Country = country, data_all=rich_data, is_startFromFirstCaseDay=True, daily=daily)
    # Consider confiromed cases and Fatalities
    data_CandF = data.values[:,0:2]
    # Consider all features
    data_multi_features = data.values

    scaled_data_b,scb = MaxMinNormalization2(data = data_CandF)
    scaled_data_c, scc = MaxMinNormalization2(data = data_multi_features)
    data_gen = TimeseriesGenerator(scaled_data_c,scaled_data_b, length = time_step, batch_size=1)
    for i in range(len(data_gen)):
        X,Y= data_gen[i]
        X_data.append(X[0])
        Y_data.append(Y[0])
else:
    if not is_single_country:
        countries = ['US',  'Canada','France','Japan','Sweden','Germany','South Africa','Australia','Italy']
        for c in countries:
            x, y = multi_country(country=c, data_all=rich_data, is_startFromFirstCaseDay=True, daily=daily, length=time_step)
            X_data = X_data+x
            Y_data = Y_data+y


X_data = np.array(X_data)
Y_data = np.array(Y_data)
# len(data_gen)
num_train = int(split*X_data.shape[0])
num_test = num_val = int((1-split)/2*X_data.shape[0])
# Split data set
# Indiex
indices = np.array(range(X_data.shape[0]))
testI = indices[-num_test:]
trainAndValI = np.setdiff1d(indices,testI)
trainI = np.random.choice(X_data[trainAndValI].shape[0],size=num_train,replace=False)
valI = np.setdiff1d(trainAndValI,trainI)
# Data
X_train = X_data[trainI]
Y_train = Y_data[trainI]
X_val = X_data[valI]
Y_val = Y_data[valI]
X_test = X_data[testI]
Y_test = Y_data[testI]
if is_single_country:
    # Date
    dates = data[time_step:].index.values
    trainDates = dates[trainI]
    valDates = dates[valI]
    testDates = dates[testI]
    trainAndValDates = dates[trainAndValI]
    #
    knownI = np.argsort(trainAndValDates)
    trainAndValDates = trainAndValDates[knownI]
    X_trainAndVal = X_data[knownI]
    Y_trainAndVal = Y_data[knownI]
epochs = 250
batch_size = 8
save_fname = os.path.join(save_dir, 'COVID19_LSTM.h5')
nfeatures = X_data.shape[2]
model = aLSTM(time_step, nfeatures=nfeatures)
print('[Model] Training Started')
print('[Model] %s epochs, %s batch size' % (epochs, batch_size))

# model.load_weights(save_fname)
callbacks = [
    EarlyStopping(monitor='val_loss', patience=50),
    ModelCheckpoint(filepath=save_fname, monitor='val_loss', save_best_only=True),
#     TensorBoard(log_dir='./logs')
]
history = model.fit(x=X_train,y=Y_train, 
                    validation_data=(X_val,Y_val),
                    validation_steps=1,
                    callbacks=callbacks,steps_per_epoch=1,epochs=epochs,verbose=1)
model.save(save_fname)
print('[Model] Training Completed. Model saved as %s' % save_fname)
printFigure(history)

# Performance in test data
results = model.evaluate(X_test, Y_test)
print('test loss, test acc:', results)
# show mutil-plots
if is_single_country :
    # Only plot figures when there is one coutry in data set. 
    Confirmed_train, Fatalities_train = plotTrainFigure(model, X_trainAndVal, Y_trainAndVal, X_test, Y_test, 
                                                    dates, trainAndValDates, testDates, scb, time_step=time_step, country=country,daily=daily)<jupyter_output>Model: "model_8"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_8 (InputLayer)         (None, 6, 8)              0         
_________________________________________________________________
lstm_13 (LSTM)               (None, 6, 128)            70144     
_________________________________________________________________
lstm_14 (LSTM)               (None, 64)                49408     
_________________________________________________________________
dense_15 (Dense)             (None, 32)                2080      
_________________________________________________________________
dropout_8 (Dropout)          (None, 32)                0         
_________________________________________________________________
dense_16 (Dense)             (None, 2)                 66        
===========================================================[...]
{'fraction_non_alphanumeric': 0.1426504862171502, 'fraction_numerical': 0.02250261191031102, 'mean_word_length': 3.660674157303371, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '411158', 'license': 'MIT', 'n_tokens_mistral': 4303, 'n_tokens_neox': 4108, 'n_words': 583, 'path': 'SimpleLSTMModel.ipynb', 'repo_name': 'nanguoyu/COVID19-Global-Forecasting', 'size': '12443'}