<jupyter_start><jupyter_text>Exploring<jupyter_code>data.head()
data['date'][0]
for station in stations:
    data.loc[(data['station'] == 'ES0691A') & (data['date'] < datetime(2014, 12, 31))]
data.groupby('station').count()
stations = data['station'].unique()
cols = [
    'date',
    'station',
    'conc_model_1', 
    'conc_model_2',
    'conc_obs_lag1',
    'target_lag1',    
    'conc_obs',
]
list(data.columns)
data = data[cols].sort_values(['date', 'station']).reset_index(drop=True)
print data.columns
data.head()
data_seq = sequences_to_columns(data, cols)
data_seq.columns
data_seq.tail()
targets = pd.read_csv(targets_path)
stations = pd.read_csv(stations_path)
print targets['target'].mean()
targets.head()
targets['target'] = 0
submission = targets['target']
submission
submission.to_csv(submission_path+'submission_zeros.csv', header=True, index=False)
stations.head()
from matplotlib import pyplot as plt
plt.plot(x=range(10), y=)<jupyter_output><empty_output><jupyter_text>Keras LSTM<jupyter_code>from airquality.models.LSTM_keras import LSTM_K
from airquality.data.prepare_data import create_model_matrix, create_ts_df
data_obs = read_obs()
data_targets = read_targets()
print data_obs.shape
target_cols = data_targets.columns[:-1]
data_obs.tail()
model_matrix = create_model_matrix(data_obs, target_cols=list(target_cols))
print model_matrix.shape
model_matrix.tail()
# split into train and test sets
train_size = int(len(model_matrix)-1)
print train_size, len(model_matrix)
trainX, testX, trainY, testY = tt_split(model_matrix, train_size, target_cols)
print trainX.shape, testX.shape
print trainY.shape, testY.shape
seq_length = 1
trainX = reshape_to_keras(trainX, seq_length)
testX = reshape_to_keras(testX, seq_length)
print trainX.shape, testX.shape
print trainY.shape, testY.shape
np.vstack((trainX, testX)).shape
param_dict = {
    'batch_size': 10,
    'seq_length': seq_length,
    'size': trainX.shape[2],
    'hidden_units': 50,
    'num_layers': 1,
    'dense_units': trainY.shape[1],
    'epochs': 5,
}

lstm = LSTM_K(**param_dict)
lstm.fit(trainX, trainY)
lstm.predict(testX)
testY<jupyter_output><empty_output><jupyter_text>validating LSTM<jupyter_code>from airquality.models.split import tt_split
model_matrix = data_seq
model_matrix.head()
model_matrix.columns
sub_samples = int(len(model_matrix)*0.2)
sub_model_matrix = model_matrix[:sub_samples]
sub_model_matrix.head()
target_cols = ['conc_obs'+'_'+station for station in stations]
target_cols
train_size = int(len(sub_model_matrix)*0.7)
trainX, testX, trainY, testY = tt_split(sub_model_matrix, train_size, target_cols)
print trainX.shape, testX.shape
print trainY.shape, testY.shape
seq_length = 1
trainX = reshape_to_keras(trainX, seq_length)
testX = reshape_to_keras(testX, seq_length)
print trainX.shape, testX.shape
print trainY.shape, testY.shape
param_dict = {
    'batch_size': 10,
    'seq_length': seq_length,
    'size': trainX.shape[2],
    'hidden_units': 50,
    'num_layers': 1,
    'dense_units': trainY.shape[1],
    'epochs': 5,
}

lstm = LSTM_K(**param_dict)
lstm.validate(trainX, trainY, testX, testY)<jupyter_output>Epoch 1/5
 - 3s - loss: 630.7661
Epoch 2/5
 - 3s - loss: 604.2950
Epoch 3/5
 - 2s - loss: 586.6212
Epoch 4/5
 - 3s - loss: 571.2174
Epoch 5/5
 - 3s - loss: 557.5864
Predicting for day 4: log-loss 224.953900
Epoch 1/10
 - 3s - loss: 543.7883
Epoch 2/10
 - 3s - loss: 529.6103
Epoch 3/10
 - 3s - loss: 513.5855
Epoch 4/10
 - 3s - loss: 498.5420
Epoch 5/10
 - 3s - loss: 482.0982
Epoch 6/10
 - 3s - loss: 463.2978
Epoch 7/10
 - 3s - loss: 445.0132
Epoch 8/10
 - 3s - loss: 427.4873
Epoch 9/10
 - 3s - loss: 411.3826
Epoch 10/10
 - 3s - loss: 395.4482
Predicting for day 8: log-loss 187.473557
Epoch 1/10
 - 3s - loss: 381.0226
Epoch 2/10
 - 3s - loss: 364.9776
Epoch 3/10
 - 3s - loss: 350.4966
Epoch 4/10
 - 3s - loss: 337.0278
Epoch 5/10
 - 3s - loss: 323.9562
Epoch 6/10
 - 3s - loss: 313.2912
Epoch 7/10
 - 3s - loss: 302.6900
Epoch 8/10
 - 3s - loss: 293.6735
Epoch 9/10
 - 3s - loss: 284.0185
Epoch 10/10
 - 3s - loss: 276.1170
Predicting for day 10: log-loss 160.138133
Epoch 1/10
 - 3s - loss: 2[...]
{'fraction_non_alphanumeric': 0.1367376573088093, 'fraction_numerical': 0.08518877057115198, 'mean_word_length': 4.518024032042724, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '348091', 'license': 'MIT', 'n_tokens_mistral': 1921, 'n_tokens_neox': 1657, 'n_words': 361, 'path': 'notebooks/01_jma_modelling.ipynb', 'repo_name': 'javiermas/BCNAirQualityDatathon', 'size': '4132'}