<jupyter_start><jupyter_text>Intial Analysis<jupyter_code>df.head(5)
df.tail(5)
df.describe()
 df.isnull().sum()
df.exchange.unique()
df.internalName.unique()
def daterange(date1, date2):
    for n in range(int ((date2 - date1).days)+1):
        yield date1 + timedelta(n)

start_dt = date(2020,1,1)
end_dt = date(2020,7,31)
weekdays = [5,6]

# gives all dates exculding weekdays
dates = [dt.strftime("%Y-%m-%d") for dt in daterange(start_dt, end_dt) if dt.weekday() not in weekdays]


all_days_in_csv = set([_.split(" ")[0] for _ in df.datetime])
all_days_between_date = dates

print ("MISSING DATES EXCLUDING WEEKENDS")
for _ in dates:
    if _ not in all_days_in_csv:
        print (_)<jupyter_output>MISSING DATES EXCLUDING WEEKENDS
2020-02-21
2020-03-10
2020-04-02
2020-04-06
2020-04-10
2020-04-14
2020-05-01
2020-05-25<jupyter_text>What does these dates mean....?<jupyter_code>from IPython.display import Image
Image(filename='01.png')
# Rolling Mean
mavg = df.close[:2000].rolling(window=100).mean()
df.close[:2000].plot()
mavg.plot()
returns = df.close[:3000] / df.close[:3000].shift(1) - 1
mavg_r = returns.rolling(window=5).mean()
returns.plot()
mavg_r.plot()
scaler = MinMaxScaler(feature_range=(-1, 1))
df.close = scaler.fit_transform(df.close.values.reshape(-1,1))
def load_data(stock, look_back, test_ratio=0.2):
    # convert to numpy array
    data_raw = stock.as_matrix() 
    data = []
    
    # convert [1,2,3,4,5] = [[1,2],[2,3],[3,4],[455]]
    for index in range(len(data_raw) - look_back): 
        data.append(data_raw[index: index + look_back])
    
    data = np.array(data);
    test_set_size = int(np.round(test_ratio*data.shape[0]));
    train_set_size = data.shape[0] - (test_set_size);
    
    x_train = data[:train_set_size,:-1]
    x_train = np.reshape(x_train, (x_train.shape[0],x_train.shape[1],1))

    y_train = data[:train_set_size,-1]
    y_train = np.reshape(y_train, (x_train.shape[0],1))
    
    x_test = data[train_set_size:,:-1]
    x_test = np.reshape(x_test, (x_test.shape[0],x_test.shape[1],1))
    
    y_test = data[train_set_size:,-1]
    y_test = np.reshape(y_test,(y_test.shape[0],1))
    
    return [x_train, y_train, x_test, y_test]
look_back = 20 
x_train, y_train, x_test, y_test = load_data(df.close, look_back)

print('x_train.shape = {}'.format(x_train.shape))
print('y_train.shape = {}'.format(y_train.shape))
print('x_test.shape = {}'.format(x_test.shape))
print('y_test.shape = {}'.format(y_test.shape))
# convert to Tensors
x_train = torch.from_numpy(x_train).type(torch.Tensor)
x_test = torch.from_numpy(x_test).type(torch.Tensor)
y_train = torch.from_numpy(y_train).type(torch.Tensor)
y_test = torch.from_numpy(y_test).type(torch.Tensor)
x_train.size()
class LSTM(nn.Module):
    def __init__(self, input_dim, hidden_dim, num_layers, output_dim):
        super().__init__()
        
        self.hidden_dim = hidden_dim
        self.num_layers = num_layers

        self.lstm = nn.LSTM(input_dim, hidden_dim, num_layers, batch_first=True)

        self.fc = nn.Linear(hidden_dim, output_dim)

    def forward(self, x):
        # print (x.shape,x.size(0))
        
        # Initialize hidden state, cell state with zeros
        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).requires_grad_()
        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_dim).requires_grad_()

        out, (hn, cn) = self.lstm(x, (h0.detach(), c0.detach()))

        # just want last time step hidden states! 
        out = self.fc(out[:, -1, :]) 
        # out.size() --> 100, 10
        return out
model = LSTM(input_dim=1, hidden_dim=32, output_dim=1, num_layers=2)

loss_fn = torch.nn.MSELoss(size_average=True)

optimiser = torch.optim.Adam(model.parameters(), lr=0.01)

print(model)


num_epochs = 100
hist = np.zeros(num_epochs)

# Number of steps to unroll
seq_dim =look_back-1  

for t in range(num_epochs):
    # Initialise hidden state
    # Don't do this if you want your LSTM to be stateful
    #model.hidden = model.init_hidden()
    
    # Forward pass
    y_train_pred = model(x_train)

    loss = loss_fn(y_train_pred, y_train)
    if t % 10 == 0 and t !=0:
        print("Epoch ", t, "MSE: ", loss.item())
    hist[t] = loss.item()

    # Zero out gradient, else they will accumulate between epochs
    optimiser.zero_grad()

    # Backward pass
    loss.backward()

    # Update parameters
    optimiser.step()


plt.show()
plt.plot(y_train_pred.detach().numpy(), label="Preds")
plt.plot(y_train.detach().numpy(), label="Data")
plt.legend()
plt.show()

plt.plot(hist, label="Training loss")
plt.legend()
# make predictions
y_test_pred = model(x_test)

# invert predictions
y_train_pred = scaler.inverse_transform(y_train_pred.detach().numpy())
y_train = scaler.inverse_transform(y_train)
y_test_pred = scaler.inverse_transform(y_test_pred.detach().numpy())
y_test = scaler.inverse_transform(y_test)

# calculate root mean squared error
trainScore = math.sqrt(mean_squared_error(y_train[:,0], y_train_pred[:,0]))
print('Train Score: %.2f RMSE' % (trainScore))
testScore = math.sqrt(mean_squared_error(y_test[:,0], y_test_pred[:,0]))
print('Test Score: %.2f RMSE' % (testScore))
trainPredictPlot = np.empty_like(df.close)

trainPredictPlot[:] = np.nan
trainPredictPlot = np.reshape(trainPredictPlot,(trainPredictPlot.shape[0],1))
print(trainPredictPlot.shape)

trainPredictPlot[look_back:len(y_train_pred)+look_back] = y_train_pred

# shift test predictions for plotting
testPredictPlot = np.empty_like(df.close)

testPredictPlot[:] = np.nan
testPredictPlot = np.reshape(testPredictPlot,(testPredictPlot.shape[0],1))

testPredictPlot[len(y_train_pred)+look_back-1:len(df.close)-1, :] = y_test_pred

# plot baseline and predictions
plt.figure()
plt.plot(scaler.inverse_transform(df.close.values.reshape(-1, 1)))
plt.plot(trainPredictPlot)
plt.plot(testPredictPlot)
plt.show()<jupyter_output>(54877, 1)
{'fraction_non_alphanumeric': 0.152317880794702, 'fraction_numerical': 0.03379181524876889, 'mean_word_length': 4.408631772268136, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '555210', 'license': 'MIT', 'n_tokens_mistral': 2414, 'n_tokens_neox': 2253, 'n_words': 402, 'path': 'Stock prediction using simple LSTM network (pytorch).ipynb', 'repo_name': 'Joish/others', 'size': '5889'}