<jupyter_start><jupyter_text>upload data please upload all.csv<jupyter_code>from google.colab import files
uploaded = files.upload()<jupyter_output><empty_output><jupyter_text>read csv<jupyter_code>def readTrain():
  train = pd.read_csv("all.csv")
  return train
#read data
train = readTrain()<jupyter_output><empty_output><jupyter_text>give data column time : 2016~2019 WS :windspeed WD: winddirect temp: tempture DTR : value(we have to validation this)<jupyter_code>trainnp = train.values
train1 = pd.DataFrame(trainnp)
train1.columns = ['time','WS','WD','temp','DTR']
train1[0:5]<jupyter_output><empty_output><jupyter_text>Completion of continuous time missing and data missing  the origial data has a lot of NAN and -1,and continuous time missing, we have to fix it<jupyter_code>def makeTime(train):
  train['time'] = pd.to_datetime(train['time'])
  grouper = pd.Grouper(key='time', freq='60T')
  res = train.groupby(grouper).first().reset_index()
  res = res.fillna(method="ffill") # propagate non-null values forward
  return res

trainTime = makeTime(train1)
trainTime[14814:14822]<jupyter_output><empty_output><jupyter_text>add featureThe original field has only four features, wind speed, time, wind direction and temperature. For our prediction of DTR, there are very few features, so we have to divide the time into months, date,day of week and hour<jupyter_code>def augFeatures(train):
  train["time"] = pd.to_datetime(train["time"])
#   train["year"] = train["time"].dt.year
  train["month"] = train["time"].dt.month
  train["date"] = train["time"].dt.day
  train["dayofweek"] = train["time"].dt.dayofweek
  train["hour"] = train["time"].dt.hour
  return train
train_Aug = augFeatures(trainTime)
train_Aug[0:5]<jupyter_output><empty_output><jupyter_text>drop time column<jupyter_code>def dropSomething(train):
  train = train.drop(["time"], axis=1)

  return train
train_drop = dropSomething(train_Aug)<jupyter_output><empty_output><jupyter_text>replace -1 to NULL<jupyter_code>trainnon = train_drop.replace({'WS':-1,'WD':-1,'temp':-1,'DTR':-1},np.nan)<jupyter_output><empty_output><jupyter_text>feature scaling<jupyter_code>def normalize(train):
  train_norm = train.apply(lambda x: (x - np.nanmean(x)) / (np.max(x) - np.min(x)))
  return train_norm
train_norm = normalize(trainnon)<jupyter_output><empty_output><jupyter_text>slice data(predicting 6 data by the first 24 data)<jupyter_code>#predicting ? data by the first ? data
def buildTrain(train, pastDay, futureDay):
  X_train, Y_train = [], []
  #cut pastDay and futureDay can get range
  for i in range(train.shape[0]-futureDay-pastDay):
    a = train.iloc[i:i+pastDay]
#     a = a.drop(["DTR"], axis=1)
    b = train.iloc[i+pastDay:i+pastDay+futureDay]["DTR"]
    #check data and drop NULL data
    if a.isnull().values.any() or b.isnull().values.any():
      continue
    else:
      X_train.append(np.array(a))
      Y_train.append(np.array(b))
  return np.array(X_train),np.array(Y_train)
X_all, Y_all = buildTrain(train_norm, 24, 6)<jupyter_output><empty_output><jupyter_text>X_all : data feature Y_all : data result<jupyter_code>print(X_all.shape)
print(Y_all.shape)<jupyter_output>(23715, 24, 8)
(23715, 6)<jupyter_text>put out some  data (no  training, no validation, just strange data)<jupyter_code>X_train = X_all[:22240]
Y_train = Y_all[:22240]
X_validate = X_all[22241:]
Y_validate = Y_all[22241:]
print(X_train.shape)<jupyter_output>(22240, 24, 8)<jupyter_text>random slice data to train data and validation data<jupyter_code>X_train, X_test, Y_train, Y_test = train_test_split(X_train, Y_train, test_size=0.15)

print(X_train.shape)
print(Y_train.shape)
print(X_test.shape)
print(Y_test.shape)<jupyter_output>(18904, 24, 8)
(18904, 6)
(3336, 24, 8)
(3336, 6)<jupyter_text>reshape (Only For Many to Many model)<jupyter_code># from 2 dimmension to 3 dimension
Y_train = Y_train[:,:,np.newaxis]
Y_test = Y_test[:,:,np.newaxis]
Y_test.shape<jupyter_output><empty_output><jupyter_text>build model<jupyter_code>#many to many(same sequence length)
def buildManyToManyModel(shape):
  model = Sequential()
  model.add(LSTM(128, input_length=shape[1], input_dim=shape[2], return_sequences=True))
  # output shape: (5, 1)
  model.add(TimeDistributed(Dense(1)))
  model.compile(loss="mae", optimizer="adam")
  model.summary()
  return model
#many to many (different sequence length)
def buildManyTodifManyModel(shape):
  model = Sequential()  
  model.add(LSTM(input_length=shape[1],input_dim=shape[2], output_dim=300, return_sequences=False))  
  model.add(Dropout(0.2))
  model.add(RepeatVector(6))
  model.add(LSTM(output_dim=300, return_sequences=True))  
  model.add(Dropout(0.2))
  model.add(TimeDistributed(Dense(1)))
#   model.add(Activation('linear'))   
  model.compile(loss='mse', optimizer='adam') 
  model.summary()
  return model
#many to one
def buildManyToOneModel(shape):
  model = Sequential()
  model.add(LSTM(300, input_length=shape[1], input_dim=shape[2], return_sequences=True))
  model.add(Dropout(0.2))
#   model.add(LSTM(256, return_sequences=True))
#   model.add(Dropout(0.2))
  model.add(LSTM(300, return_sequences=False))
  model.add(Dropout(0.2))
  # output shape: (1, 1)
#   model.add(Dense(512))
#   model.add(Dropout(0.2))
#   model.add(Dense(256))
#   model.add(Dropout(0.2))
#   model.add(Dense(128))
#   model.add(Dropout(0.2))
  model.add(Dense(1))
  adam = keras.optimizers.Adam(lr = 0.0001)
  model.compile(loss="mae", optimizer=adam)
  model.summary()
  return model
#many to many(same sequence length)
# model = buildManyToManyModel(X_train.shape)
#many to many(different sequence length)
model = buildManyTodifManyModel(X_train.shape)
# many to one
# model = buildManyToOneModel(X_train.shape)
callback = EarlyStopping(monitor="val_loss",min_delta=0.0001, patience=15,verbose=1, mode="auto")
history = model.fit(X_train, Y_train, epochs=50, batch_size=128, validation_data=(X_test, Y_test),callbacks=[callback]) #callbacks=[callback]<jupyter_output>/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: UserWarning: The `input_dim` and `input_length` arguments in recurrent layers are deprecated. Use `input_shape` instead.
  This is separate from the ipykernel package so we can avoid doing imports until
/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:3: UserWarning: Update your `LSTM` call to the Keras 2 API: `LSTM(return_sequences=False, input_shape=(24, 8), units=300)`
  This is separate from the ipykernel package so we can avoid doing imports until
/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:6: UserWarning: Update your `LSTM` call to the Keras 2 API: `LSTM(return_sequences=True, units=300)`<jupyter_text>Show training history<jupyter_code>import matplotlib.pyplot as plt

def show_train_history(train_history,train,validation):
    plt.plot(train_history.history[train])
    plt.plot(train_history.history[validation])
    plt.title('Train History')
    plt.ylabel(train)
    plt.xlabel('Epoch')
    plt.legend(['train', 'validation'], loc='upper left')
    plt.ylim([0,0.01])
    plt.show()

show_train_history(history,'loss','val_loss')<jupyter_output><empty_output><jupyter_text>Load strange data and view forecast results<jupyter_code># results = np.argmax(model.predict(X_test)) #forcast
results = (model.predict(X_validate)) #forcast
print(results[0])
print(Y_validate[0])<jupyter_output>[[-0.13010931]
 [-0.11443662]
 [-0.11148709]
 [-0.10864223]
 [-0.10690326]
 [-0.10580356]]
[-0.05752437 -0.08640642 -0.12817568 -0.18964312 -0.18367111 -0.31524379]<jupyter_text>Calculate the maximum and average error values (for many to on)<jupyter_code>d = 0
list5 = []
list6 = []
list7 = []

for i in range(Y_validate.shape[0]):
  #a:test data result  b:predict data
  a = Y_validate[i]*(np.max(trainnon['DTR']) - np.min(trainnon['DTR'])) + np.nanmean(trainnon['DTR'])
  b = results[i]*(np.max(trainnon['DTR']) - np.min(trainnon['DTR'])) + np.nanmean(trainnon['DTR'])
  list6.append(a)
  list7.append(b)
  e = abs(b-a)
  g = 0
  list5.append(e)
  global d
  d += e
print('validation')
print('Mean error value:{}'.format(d[0]/len(list6)))  
print('Maximum error value:{}'.format(max(list5)[0]))<jupyter_output><empty_output><jupyter_text>Calculate the maximum and average error values (for many to many)<jupyter_code>d = 0
list5 = []
list6 = []
list7 = []
list8 = []
list9 = []

for i in range(Y_validate.shape[0]):

  a = Y_validate[i]*(np.max(trainnon['DTR']) - np.min(trainnon['DTR'])) + np.mean(trainnon['DTR'])
  b = results[i]*(np.max(trainnon['DTR']) - np.min(trainnon['DTR'])) + np.mean(trainnon['DTR'])
  list8.append(np.mean(a))
  list9.append(np.mean(b))
  for h in range(6):
    list6.append(a[h])
    list7.append(b[h][0])
    list5.append(abs(a[h]-b[h][0]))

print('validation')
print('Mean error value:{}'.format(np.sum(list5)/len(list5)))
print('Maximum error value:{}'.format(max(list5)))<jupyter_output>validation
Mean error value:224.62275263282098
Maximum error value:1062.7244726562403<jupyter_text>Draw a comparison of predicted and results(for many to on)<jupyter_code>#FOR　many 2 one
plt.plot(listy6[200:300])
plt.plot(list7[200:300])
plt.title('validation')
plt.ylabel('DTR')
plt.xlabel('data')
plt.legend(['result', 'predict'], loc='upper left')
plt.ylim([500,2500])
plt.show()<jupyter_output><empty_output><jupyter_text>Draw a comparison of predicted and results(for many to many)<jupyter_code># For many 2 many
plt.plot(list8[400:500])
plt.plot(list9[400:500])
plt.title('validation')
plt.ylabel('DTR')
plt.xlabel('data')
plt.legend(['result', 'predict'], loc='upper left')
plt.ylim([500,2500])
plt.show()<jupyter_output><empty_output><jupyter_text>save model<jupyter_code>model.save('Keras_DTR_LSTM.h5')<jupyter_output><empty_output><jupyter_text>download model<jupyter_code>from google.colab import files

files.download('Keras_DTR_LSTM.h5')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15571776155717762, 'fraction_numerical': 0.044201135442011354, 'mean_word_length': 6.559386973180077, 'pattern_counts': {'":': 0, '<': 83, '<?xml version=': 0, '>': 83, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '586900', 'license': 'MIT', 'n_tokens_mistral': 3989, 'n_tokens_neox': 3685, 'n_words': 684, 'path': 'LSTM_different sequence length.ipynb', 'repo_name': 'jerry800416/Keras_LSTM_different_sequence_length', 'size': '9864'}