<jupyter_start><jupyter_text>Data prepartion<jupyter_code>import glob
import numpy as np
import xarray as xr


netcdf_list=glob.glob('/home/sst/2020*.nc')

nc_cont= []
for file in netcdf_list:
    DS = xr.open_dataset(file)
    sst =DS['analysed_sst']
    sst = sst.sel(lat = slice(16, 20.5),lon = slice(69,72.5))
    sst = sst.data
    sst = np.nan_to_num(sst)
    feb.append(sst)
   
data = np.vstack(feb)
with open('/home/input_data.npy', 'wb') as f:
    np.save(f, data)
    
print(data.shape)
import numpy as np
from sklearn.preprocessing import MinMaxScaler

sst = np.load('input_data1.npy')

y = sst.copy()

y = np.nan_to_num(y)

#print("let's take a subset of 500 X 500 and make train-test split")
data_train = np.array(y)
data_train = data_train.reshape(len(data_train),158301)
#data_test = np.array(y[600:])

scaler = MinMaxScaler(feature_range=(0,1))
data_train = scaler.fit_transform(data_train)<jupyter_output><empty_output><jupyter_text>Application of LSTM in imagery forecast<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import LSTM
from keras.layers import Dropout
from keras.callbacks import EarlyStopping
from keras.callbacks import ModelCheckpoint
from keras.models import load_model

X_train = []
y_train = []

for i in range(10, 363):
    X_train.append(data_train[i-10:i])
    y_train.append(data_train[i:i+3])
X_train, y_train = np.array(X_train), np.array(y_train)
X_train = np.reshape(X_train, (X_train.shape[0], X_train.shape[1],158301))
print('Shape of X_train is:',X_train.shape)
print('Shape of y_train is:',y_train.shape)
print("Initialising the RNN")

regressor = Sequential()

# Layer 1
regressor.add(LSTM(units = 10,activation = 'relu', return_sequences = True, input_shape = (X_train.shape[1], 210166)))
# layer 2
regressor.add(LSTM(units = 10, return_sequences = True))
# layer 3
regressor.add(LSTM(units = 10, return_sequences = True))
#regressor.add(Dropout(0.2))
# layer 4
regressor.add(LSTM(units = 10))
#regressor.add(Dropout(0.2))

# Adding the output layer
regressor.add(Dense(units = 630498))

#model apply
regressor.compile(optimizer = 'adam', loss = 'mean_squared_error',metrics = 'mean_absolute_error')

y_train = np.reshape(y_train,(353,630498))
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1,patience=3)
mc = ModelCheckpoint('model_output.h5', monitor='val_loss', mode='min',save_best_only=True, verbose=1)
model = regressor.fit(X_train, y_train,validation_split=0.2, epochs = 10, batch_size = 5,callbacks = [mc])<jupyter_output><empty_output><jupyter_text>Model evaluation<jupyter_code>test = np.load('test_data.npy')
model_sst  = load_model('model_output.h5')
array = test[-23:-13]
scaler = MinMaxScaler(feature_range=(0,1))
array = array.reshape(len(array),210166)
array = scaler.fit_transform(array)
array = np.reshape(array,(1,10,210166))
forecast = model_sst.predict(array)
day0 = forecast[0,0:210166]
day1 = forecast[0,210166:420332]
day2 = forecast[0,-210166:]
day0 = day0.reshape(1,210166)
day1 = day1.reshape(1,210166)
day2 = day2.reshape(1,210166)
day0 = scaler.inverse_transform(day0)
day1 = scaler.inverse_transform(day1)
day2 = scaler.inverse_transform(day2)
day0 = day0.reshape(466,451)
day1 = day1.reshape(466,451)
day2 = day2.reshape(466,451)

rmse0 = np.sqrt(np.mean((test[-13]-day0)**2))
rmse1= np.sqrt(np.mean((test[-12]-day1)**2))
rmse2= np.sqrt(np.mean((test[-11]-day2)**2))
print('RMSE for day0:',rmse0)
print('RMSE for day1:',rmse1)
print('RMSE for day2:',rmse2)
test.shape
inputs = y[len(y)-len(data_test)-10:]
#inputs = inputs[:,:500,:500]
inputs = np.reshape(inputs,(len(inputs),210166))
inputs = scaler.transform(inputs)
x_test=[]
for i in range(10,140):
    x_test.append(inputs[i-10:i])

x_test = np.array(x_test)
x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 210166))
predicted_ch = regressor.predict(x_test)
predicted = scaler.inverse_transform(predicted_ch)

data_test = np.reshape(data_test,(130,210166))
rmse = np.sqrt(np.mean(predicted-data_test)**2)
print('RMSE of the model is: ',rmse)
print(model.history.keys())
plt.plot(model.history['loss'])
plt.plot(model.history['val_loss'])
plt.title('model performance')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'validation'], loc='upper right')
plt.show()
model_sst  = load_model('model_output.h5')
forecast = model_sst.predict(x_test)
predicted = scaler.inverse_transform(forecast)
rmse = np.sqrt(np.mean(predicted-data_test)**2)
print('RMSE of the model is: ',rmse)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15676855895196506, 'fraction_numerical': 0.05829694323144105, 'mean_word_length': 6.6096345514950166, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '123784', 'license': 'MIT', 'n_tokens_mistral': 1981, 'n_tokens_neox': 1812, 'n_words': 324, 'path': 'apply_ml_images_forecast.ipynb', 'repo_name': 'nishadhka/ml_learning', 'size': '4580'}