<jupyter_start><jupyter_text>Generate Sine Wave<jupyter_code>np.random.seed(2)

T = 20
L = 1000
N = 100
x = np.empty((N, L), 'int64')
x[:] = np.array(range(L)) + np.random.randint(-4 * T, 4 * T, N).reshape(N, 1)
x.shape
data = np.sin(x / 1.0 / T).astype('float64')
torch.save(data, open('traindata.pt', 'wb'))
data.shape
plt.plot(data[5,: ])<jupyter_output><empty_output><jupyter_text>To use the data in the model, .chunk(train_input.size(1), dim=1) splits it into slices 1, 2, ..., t, ..., L.  Model   Train and Test sets<jupyter_code># set random seed to 0
np.random.seed(0)
torch.manual_seed(0)
# load data and make training set
data = torch.load('traindata.pt')

# Train dataset
train_input = torch.from_numpy(data[3:, :-1])
train_target = torch.from_numpy(data[3:, 1:])

# Test dataset
test_input = torch.from_numpy(data[:3, :-1])
test_target = torch.from_numpy(data[:3, 1:])
data.shape
plt.plot(train_input[96].numpy())
print('Train input:', train_input.size(), ', Train target:', train_target.size())
list(train_input.chunk(train_input.size(1), dim=1))[0].size()<jupyter_output><empty_output><jupyter_text>Build the model<jupyter_code>class Sequence(nn.Module):
    def __init__(self):
        super(Sequence, self).__init__()
        self.lstm1 = nn.LSTMCell(1, 51)
        self.lstm2 = nn.LSTMCell(51, 51)
        self.linear = nn.Linear(51, 1)

    def forward(self, input, future = 0):
        outputs = []
        h_t = torch.zeros(input.size(0), 51, dtype=torch.double)
        c_t = torch.zeros(input.size(0), 51, dtype=torch.double)
        h_t2 = torch.zeros(input.size(0), 51, dtype=torch.double)
        c_t2 = torch.zeros(input.size(0), 51, dtype=torch.double)

        for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
            h_t, c_t = self.lstm1(input_t, (h_t, c_t))
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))
            output = self.linear(h_t2)
            outputs += [output]
        for i in range(future):# if we should predict the future
            h_t, c_t = self.lstm1(output, (h_t, c_t))
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))
            output = self.linear(h_t2)
            outputs += [output]
        outputs = torch.stack(outputs, 1).squeeze(2)
        return outputs
# build the model
seq = Sequence()
seq.double()
criterion = nn.MSELoss()

# use LBFGS as optimizer since we can load the whole data to train
optimizer = optim.LBFGS(seq.parameters(), lr=0.8)

# optimizer = optim.Adam(seq.parameters(), lr=0.1, weight_decay=1e-3)<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>#begin to train
for i in tqdm_notebook(range(20), desc='Epoch:'):
    
    def closure():
        optimizer.zero_grad()
        out = seq(train_input)
        loss = criterion(out, train_target)
        print('loss:', loss.item())
        loss.backward()
        return loss
    optimizer.step(closure)
    # begin to predict, no need to track gradient here
    with torch.no_grad():
        future = 1000
        pred = seq(test_input, future=future)
        loss = criterion(pred[:, :-future], test_target)
        print('test loss:', loss.item())
        y = pred.detach().numpy()
    # draw the result
    plt.figure(figsize=(30,10))
    plt.title('Predict future values for time sequences\n(Dashlines are predicted values)',
              fontsize=30)
    plt.xlabel('x', fontsize=20)
    plt.ylabel('y', fontsize=20)
    plt.xticks(fontsize=20)
    plt.yticks(fontsize=20)
    def draw(yi, color):
        plt.plot(np.arange(train_input.size(1)),
                 yi[:train_input.size(1)], color, linewidth = 2.0)
        plt.plot(np.arange(train_input.size(1), train_input.size(1) + future), 
                 yi[train_input.size(1):], color + ':', linewidth = 2.0)
    draw(y[0], 'r')
    draw(y[1], 'g')
    draw(y[2], 'b')
    plt.show()<jupyter_output><empty_output><jupyter_text>Trends and pricesSimulate price paths according to <NAME>., <NAME>., & <NAME>. (2010). Trend following trading under a regime switching model. SIAM Journal on Financial Mathematics, 1(1), 780-810.<jupyter_code>import pandas as pd
from pandas import DataFrame, Series
pd.set_option('display.max_rows', 6)
def sim_path(N=1000, P=np.array([[0.95, 0.05], [0.1, 0.9]]),
             mu=[0.05, 0.05], sigma=0.35, S0=1, dt=1/250):
    
    # State simulation
    state = np.random.randint(0, 2)
    states = [state]
    for t in range(N - 1):
        state = int(np.random.multinomial(1, P[state, :]).argmax())
        states.append(state)
    
    # Price simulation
    drift = (np.array([mu[s] for s in states[1:]]) - 0.5 * sigma**2) * dt
    wiener = np.random.randn(N - 1) * sigma * np.sqrt(dt)

    S = S0 * np.concatenate((np.ones(1), np.exp(np.cumsum(drift + wiener))))
    
    res_df = DataFrame(data=dict(Close=S, State=states))
    res_df = res_df.assign(Date = pd.date_range(start=datetime(2018, 1, 1),
                                                periods=N, freq='D')).set_index('Date')
    res_df = res_df.assign(Ch = res_df.Close.pct_change().add(1).apply('log'))
    res_df = res_df.assign(Ch2 = res_df.Ch.pow(2),
                           Ch3 = res_df.Ch.pow(3),
                           Ch4 = res_df.Ch.pow(4)).fillna(0)

    return res_df
price_df = sim_path()
price_df[['Close', 'State']].plot(figsize=(10, 4), alpha = 0.5);
price_df<jupyter_output><empty_output><jupyter_text>Only Price Train and test setsThe input data for an LSTM cell in pytorch must be of dimension:* `(seq_len, batch, input_size) = (seq_len, N_samples, 1)`.<jupyter_code>N_samples = 100

paths = [sim_path() for i in range(N_samples)]
inputs = np.array([p.Close.values for p in paths])
targets = np.array([p.State.values for p in paths])
inputs = torch.from_numpy(inputs)
targets = torch.from_numpy(targets)
inputs.size(), targets.size()
list(inputs.chunk(inputs.size(1), dim=1))[0].size()<jupyter_output><empty_output><jupyter_text>Build the model<jupyter_code>class Sequence(nn.Module):
    def __init__(self):
        super(Sequence, self).__init__()
        self.lstm1 = nn.LSTMCell(1, 51)
        self.lstm2 = nn.LSTMCell(51, 51)
        self.linear = nn.Linear(51, 2)
        self.sigmoid = nn.Sigmoid()

    def forward(self, input):
        outputs = []
        h_t = torch.zeros(input.size(0), 51, dtype=torch.double)
        c_t = torch.zeros(input.size(0), 51, dtype=torch.double)
        h_t2 = torch.zeros(input.size(0), 51, dtype=torch.double)
        c_t2 = torch.zeros(input.size(0), 51, dtype=torch.double)

        for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
            h_t, c_t = self.lstm1(input_t, (h_t, c_t))
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))
            output = self.sigmoid(self.linear(h_t2))
            outputs += [output]

        outputs = torch.stack(outputs, 1).squeeze(2)
        return outputs
# build the model
seq = Sequence()
seq.double()
criterion = nn.CrossEntropyLoss()

# use LBFGS as optimizer since we can load the whole data to train
# optimizer = optim.LBFGS(seq.parameters(), lr=0.8)

optimizer = optim.Adam(seq.parameters())
out = seq(inputs)
out
out.view(-1, 2).size()
targets.view(-1).size()
out.size(), targets.size()
criterion(out.view(-1, 2), targets.view(-1))<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>#begin to train
for i in tqdm_notebook(range(20), desc='Epoch:'):
    
    def closure():
        optimizer.zero_grad()
        out = seq(inputs)
        loss = criterion(out.view(-1, 2), targets.view(-1))
        print('loss:', loss.item())
        loss.backward()
        return loss
    optimizer.step(closure)
    # begin to predict, no need to track gradient here
#     with torch.no_grad():
#         future = 1000
#         pred = seq(test_input, future=future)
#         loss = criterion(pred[:, :-future], test_target)
#         print('test loss:', loss.item())
#         y = pred.detach().numpy()
outout = seq(inputs)
out<jupyter_output><empty_output><jupyter_text>Prices and changes  Train and test setsThe input data for an LSTM cell in pytorch must be of dimension:* `(seq_len, batch, input_size) = (seq_len, N_samples, 5)`.<jupyter_code>N_samples = 100

paths = [sim_path() for i in range(N_samples)]
inputs = np.array([p.drop('State', axis=1).values for p in paths])
targets = np.array([p.State.values for p in paths])
inputs = torch.from_numpy(inputs)
targets = torch.from_numpy(targets)
inputs.size(), targets.size()
list(inputs.chunk(inputs.size(1), dim=1))[0].view(N_samples, -1).size()<jupyter_output><empty_output><jupyter_text>Build the model<jupyter_code>class Sequence(nn.Module):
    def __init__(self):
        super(Sequence, self).__init__()
        self.lstm1 = nn.LSTMCell(5, 51)
        self.lstm2 = nn.LSTMCell(51, 51)
        self.linear = nn.Linear(51, 2)
        self.sigmoid = nn.Sigmoid()

    def forward(self, input):
        outputs = []
        h_t = torch.zeros(input.size(0), 51, dtype=torch.double)
        c_t = torch.zeros(input.size(0), 51, dtype=torch.double)
        h_t2 = torch.zeros(input.size(0), 51, dtype=torch.double)
        c_t2 = torch.zeros(input.size(0), 51, dtype=torch.double)

        for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
            h_t, c_t = self.lstm1(input_t.view(N_samples, -1), (h_t, c_t))
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))
            output = self.sigmoid(self.linear(h_t2))
            outputs += [output]

        outputs = torch.stack(outputs, 1).squeeze(2)
        return outputs
# build the model# build 
seq = Sequence()
seq.double()
criterion = nn.CrossEntropyLoss()

# use LBFGS as optimizer since we can load the whole data to train
# optimizer = optim.LBFGS(seq.parameters(), lr=0.1)

optimizer = optim.Adam(seq.parameters())
out = seq(inputs)
out
out.size(), targets.size()
out.view(-1, 2).size()
targets.view(-1).size()
criterion(out.view(-1, 2), targets.view(-1))<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>#begin to train
for i in tqdm_notebook(range(20), desc='Epoch:'):
    
    def closure():
        optimizer.zero_grad()
        out = seq(inputs)
        loss = criterion(out.view(-1, 2), targets.view(-1))
        print('loss:', loss.item())
        loss.backward()
        return loss
    optimizer.step(closure)
    # begin to predict, no need to track gradient here
#     with torch.no_grad():
#         future = 1000
#         pred = seq(test_input, future=future)
#         loss = criterion(pred[:, :-future], test_target)
#         print('test loss:', loss.item())
#         y = pred.detach().numpy()
out = seq(inputs)
out<jupyter_output><empty_output><jupyter_text>Support and Resistance<jupyter_code>def add_support_resistance(df, fast=30, slow=120):
    return df.assign(Resistance_fast = df.Close.rolling(30).max(),
                     Resistance_slow = df.Close.rolling(120).max(),
                     Support_fast = df.Close.rolling(30).min(),
                     Support_slow = df.Close.rolling(120).min())
price_df = add_support_resistance(sim_path())

price_df[['Close', 'Support_fast', 'Resistance_fast',
          'Support_slow', 'Resistance_slow']].plot(figsize=(12, 4));
price_df<jupyter_output><empty_output><jupyter_text>Train and test sets<jupyter_code>N_samples = 100

input_variables = ['Close']
target_variables = ['Resistance_fast', 'Resistance_slow',
                    'Support_fast', 'Support_slow']

paths = [add_support_resistance(sim_path()) for i in range(N_samples)]
inputs = np.array([p[input_variables].values for p in paths], dtype=np.double)
targets = np.array([p[target_variables].values for p in paths], dtype=np.double)
inputs = torch.from_numpy(inputs)
targets = torch.from_numpy(targets)
inputs.size(), targets.size()
# list(inputs.chunk(inputs.size(1), dim=1))[0].view(N_samples, -1).size()<jupyter_output><empty_output><jupyter_text>Build the model<jupyter_code>class Sequence(nn.Module):
    def __init__(self, h0=1, h1=51, h2=51, hL=1):
        super(Sequence, self).__init__()
        self.h0 = h0
        self.h1 = h1
        self.h2 = h2
        self.hL = hL
        self.lstm1 = nn.LSTMCell(h0, h1)
        self.lstm2 = nn.LSTMCell(h1, h2)
        self.linear = nn.Linear(h2, hL)

    def forward(self, input):
        outputs = []
        h_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        c_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        h_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)
        c_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)

        for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
            h_t, c_t = self.lstm1(input_t.view(N_samples, -1), (h_t, c_t))
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))
            output = self.linear(h_t2)
            outputs += [output]

        outputs = torch.stack(outputs, 1).squeeze(2)
        return outputs

    
class Sequence3(nn.Module):
    def __init__(self, h0=1, h1=10, h2=10, h3=10, hL=1):
        super(Sequence3, self).__init__()
        self.h0 = h0
        self.h1 = h1
        self.h2 = h2
        self.h3 = h3
        self.hL = hL
        self.lstm1 = nn.LSTMCell(h0, h1)
        self.lstm2 = nn.LSTMCell(h1, h2)
        self.lstm3 = nn.LSTMCell(h2, h3)
        self.linear = nn.Linear(h3, hL)

    def forward(self, input):
        outputs = []
        h_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        c_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        h_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)
        c_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)
        h_t3 = torch.ones(input.size(0), self.h3, dtype=torch.double)
        c_t3 = torch.zeros(input.size(0), self.h3, dtype=torch.double)

        for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
            h_t, c_t = self.lstm1(input_t.view(N_samples, -1), (h_t, c_t))
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))
            h_t3, c_t3 = self.lstm3(h_t2, (h_t3, c_t3))
            output = self.linear(h_t3)
            outputs += [output]

        outputs = torch.stack(outputs, 1).squeeze(2)
        return outputs

class Sequence4(nn.Module):
    def __init__(self, h0=1, h1=5, h2=5, h3=5, h4=5, hL=1):
        super(Sequence4, self).__init__()
        self.h0 = h0
        self.h1 = h1
        self.h2 = h2
        self.h3 = h3
        self.h4 = h4
        self.hL = hL
        self.lstm1 = nn.LSTMCell(h0, h1)
        self.lstm2 = nn.LSTMCell(h1, h2)
        self.lstm3 = nn.LSTMCell(h2, h3)
        self.lstm4 = nn.LSTMCell(h3, h4)
        self.linear = nn.Linear(h4, hL)

    def forward(self, input):
        outputs = []
        h_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        c_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        h_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)
        c_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)
        h_t3 = torch.ones(input.size(0), self.h3, dtype=torch.double)
        c_t3 = torch.zeros(input.size(0), self.h3, dtype=torch.double)
        h_t4 = torch.ones(input.size(0), self.h4, dtype=torch.double)
        c_t4 = torch.zeros(input.size(0), self.h4, dtype=torch.double)

        for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
            h_t, c_t = self.lstm1(input_t.view(N_samples, -1), (h_t, c_t))
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))
            h_t3, c_t3 = self.lstm3(h_t2, (h_t3, c_t3))
            h_t4, c_t4 = self.lstm4(h_t3, (h_t4, c_t4))
            output = self.linear(h_t4)
            outputs += [output]

        outputs = torch.stack(outputs, 1).squeeze(2)
        return outputs
# build the model
seq = Sequence4(hL=4)#Sequence(h0=1, h1=20, h2=10, hL=4)
seq.double()
mse = nn.MSELoss(reduce=False)
def criterion(y_hat, y_true):
    loss = mse(y_hat[:, 119:, :], y_true[:, 119:, :]).\
        div(y_true[:, 119:, :]).mean()
    return loss
out = seq(inputs)
out
out.size(), targets.size()
criterion(out, targets)<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>losses = []
optimizer = optim.Adam(seq.parameters(), lr=7e-5)
for i in tqdm_notebook(range(2500), desc='Epoch:'):
    
    def closure():
        optimizer.zero_grad()
        out = seq(inputs)
        loss = criterion(out, targets)
        losses.append(loss.detach().numpy())
        loss.backward()
        return loss
    optimizer.step(closure)
    
    if i % 10 == 0:
        print('i:', i, ', loss:', losses[-1])
    # begin to predict, no need to track gradient here
#     with torch.no_grad():
#         future = 1000
#         pred = seq(test_input, future=future)
#         loss = criterion(pred[:, :-future], test_target)
#         print('test loss:', loss.item())
#         y = pred.detach().numpy()
plt.plot(np.array(losses[-1000:]))<jupyter_output><empty_output><jupyter_text>Train sample<jupyter_code>with torch.no_grad():
    out = seq(inputs)
out.size()
criterion(out, targets).item()
j = np.random.randint(0, N_samples)
y_hat = out.detach().numpy()
y_hat_df = DataFrame(data=y_hat[j,...], columns=['rf', 'rs', 'sf', 'ss'])
res_j = pd.concat((paths[j], y_hat_df), axis=1)
res_j[['Close', 'Support_slow', 'ss']].plot(figsize=(12, 5));<jupyter_output><empty_output><jupyter_text>Test sample<jupyter_code>test_samples = 20

input_variables = ['Close']
target_variables = ['Resistance_fast', 'Resistance_slow',
                    'Support_fast', 'Support_slow']

test_paths = [add_support_resistance(sim_path(N=500)) for i in range(N_samples)]
test_inputs = np.array([p[input_variables].values for p in test_paths])
test_targets = np.array([p[target_variables].values for p in test_paths])
test_inputs = torch.from_numpy(test_inputs)
test_targets = torch.from_numpy(test_targets)
test_inputs.size()
with torch.no_grad():
    test_out = seq(test_inputs)
test_out.size()
criterion(test_out, test_targets).item()
j = np.random.randint(0, test_samples)
y_hat = test_out.detach().numpy()
y_hat_df = DataFrame(data=y_hat[j,...], columns=['rf', 'rs', 'sf', 'ss'])
res_j = pd.concat((test_paths[j], y_hat_df), axis=1)
res_j[['Close', 'Support_slow', 'ss']].plot(figsize=(12, 5));<jupyter_output><empty_output><jupyter_text>Trend TO DO:* Data set with Trend in -1, 0, 1.<jupyter_code>from trading_strategies import *
from utils import *
def add_trend(df, sd=20):
    df = df.assign(High=df.Close, Low=df.Close, Open=df.Close)
    df = find_trends(df, sd=sd)
    df.Trend = df.Trend.clip_lower(0)
    return df
price_df = add_trend(sim_path())
plot_trends(price_df)
price_df.groupby('n_Trend').first().Ratio.values#.sort_values().values<jupyter_output><empty_output><jupyter_text>Train and test sets  Train sample<jupyter_code>N_samples = 100
input_variables = ['Close', 'Ch', 'Ch2', 'Ch3', 'Ch4']
target_variables = ['Trend']
n_features = len(input_variables)

paths = [add_trend(sim_path(N=500)) for i in range(N_samples)]
inputs = np.array([p[input_variables].values for p in paths], dtype=np.double)
targets = np.array([p[target_variables].values for p in paths], dtype=np.double)
inputs = torch.from_numpy(inputs)
targets = torch.from_numpy(targets)
inputs.size(), targets.size()
list(inputs.chunk(inputs.size(1), dim=1))[100].view(N_samples, -1).equal(
    list(inputs.chunk(inputs.size(1), dim=1))[100].view(-1, n_features)
)<jupyter_output><empty_output><jupyter_text>Test sample<jupyter_code>test_samples = 100
test_paths = [add_trend(sim_path(N=500)) for i in range(test_samples)]
test_inputs = np.array([p[input_variables].values for p in test_paths])
test_targets = np.array([p[target_variables].values for p in test_paths])
test_inputs = torch.from_numpy(test_inputs)
test_targets = torch.from_numpy(test_targets)
test_inputs.size()<jupyter_output><empty_output><jupyter_text>Build the model<jupyter_code>class Sequence(nn.Module):
    def __init__(self, h0=1, h1=51, h2=51, hL=1):
        super(Sequence, self).__init__()
        self.h0 = h0
        self.h1 = h1
        self.h2 = h2
        self.hL = hL
        self.lstm1 = nn.LSTMCell(h0, h1)
        self.lstm2 = nn.LSTMCell(h1, h2)
        self.linear = nn.Linear(h2, hL)
        self.sigmoid = nn.Sigmoid()

    def forward(self, input):
        outputs = []
        h_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        c_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        h_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)
        c_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)

        for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
            h_t, c_t = self.lstm1(input_t.view(-1, self.h0), (h_t, c_t))
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))
            output = self.sigmoid(self.linear(h_t2))
            outputs += [output]

        outputs = torch.stack(outputs, 1)
        return outputs

class Sequence3(nn.Module):
    def __init__(self, h0=1, h1=10, h2=10, h3=10, hL=1):
        super(Sequence3, self).__init__()
        self.h0 = h0
        self.h1 = h1
        self.h2 = h2
        self.h3 = h3
        self.hL = hL
        self.lstm1 = nn.LSTMCell(h0, h1)
        self.lstm2 = nn.LSTMCell(h1, h2)
        self.lstm3 = nn.LSTMCell(h2, h3)
        self.linear = nn.Linear(h3, hL)
        self.sigmoid = nn.Sigmoid()

    def forward(self, input):
        outputs = []
        h_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        c_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        h_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)
        c_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)
        h_t3 = torch.ones(input.size(0), self.h3, dtype=torch.double)
        c_t3 = torch.zeros(input.size(0), self.h3, dtype=torch.double)

        for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
            h_t, c_t = self.lstm1(input_t.view(-1, self.h0), (h_t, c_t))
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))
            h_t3, c_t3 = self.lstm3(h_t2, (h_t3, c_t3))
            output = self.sigmoid(self.linear(h_t3))
            outputs += [output]

        outputs = torch.stack(outputs, 1)
        return outputs

class Sequence4(nn.Module):
    def __init__(self, h0=1, h1=5, h2=5, h3=5, h4=5, hL=1):
        super(Sequence4, self).__init__()
        self.h0 = h0
        self.h1 = h1
        self.h2 = h2
        self.h3 = h3
        self.h4 = h4
        self.hL = hL
        self.lstm1 = nn.LSTMCell(h0, h1)
        self.lstm2 = nn.LSTMCell(h1, h2)
        self.lstm3 = nn.LSTMCell(h2, h3)
        self.lstm4 = nn.LSTMCell(h3, h4)
        self.linear = nn.Linear(h4, hL)
        self.sigmoid = nn.Sigmoid()

    def forward(self, input):
        outputs = []
        h_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        c_t = torch.zeros(input.size(0), self.h1, dtype=torch.double)
        h_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)
        c_t2 = torch.zeros(input.size(0), self.h2, dtype=torch.double)
        h_t3 = torch.ones(input.size(0), self.h3, dtype=torch.double)
        c_t3 = torch.zeros(input.size(0), self.h3, dtype=torch.double)
        h_t4 = torch.ones(input.size(0), self.h4, dtype=torch.double)
        c_t4 = torch.zeros(input.size(0), self.h4, dtype=torch.double)

        for i, input_t in enumerate(input.chunk(input.size(1), dim=1)):
            h_t, c_t = self.lstm1(input_t.view(-1, self.h0), (h_t, c_t))
            h_t2, c_t2 = self.lstm2(h_t, (h_t2, c_t2))
            h_t3, c_t3 = self.lstm3(h_t2, (h_t3, c_t3))
            h_t4, c_t4 = self.lstm4(h_t3, (h_t4, c_t4))
            output = self.sigmoid(self.linear(h_t4))
            outputs += [output]

        outputs = torch.stack(outputs, 1)
        return outputs
# build the model
seq = Sequence3(h0=5, h1 = 10, h2 = 10, h3=10, hL=1)#, h1=10, h2=8, h3=6, h4=3)
seq.double()
out = seq(inputs)
out.size()
out.size(), targets.size()
warmup = 60

def loss_fun(y_hat, y_true):
    return torch.mean(- y_true * y_hat.log() - \
              (1 - y_true) * (1 - y_hat).log())

def criterion(y_hat, y_true, warmup=90):
    loss = loss_fun(y_hat[:, warmup:, :], y_true[:, warmup:, :])
    return loss
criterion(out, targets)
with torch.no_grad():
    test_out = seq(test_inputs)
test_out.size()
criterion(test_out, test_targets).item()<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>batch_size = 200
time_slip = 5
losses = []
test_losses = []
optimizer = optim.Adam(seq.parameters(), lr=1e-4)
# for e in tqdm_notebook(range(100), desc='Epoch:'):
    
#     def closure():
#         optimizer.zero_grad()
#         out = seq(inputs)
#         loss = criterion(out, targets)
#         losses.append(loss.detach().numpy())
#         loss.backward()
#         return loss
#     optimizer.step(closure)
    
#     if e % 10 == 0:
#         with torch.no_grad():
#             out = seq(inputs)
#             loss = criterion(out, targets)
#             test_out = seq(test_inputs)
#             test_loss = criterion(test_out, test_targets)
#         print('Epoch: ', e, ', training loss: ', loss.item(), ', test loss: ', test_loss.item(), sep='')
for e in tqdm_notebook(range(200), desc='Epoch:'):
    
    idx = np.random.permutation(np.arange(N_samples))
    
    for i in tqdm_notebook(range(1, N_samples // batch_size + 1), desc='Mini-batch:', leave=False):
        t_slip = np.random.randint(1, time_slip + 1)
        def closure():
            optimizer.zero_grad()
            out = seq(inputs[idx[np.arange((i - 1) * batch_size, i * batch_size)], t_slip:-t_slip, ...])
            loss = criterion(out, targets[idx[np.arange((i - 1) * batch_size, i * batch_size)], t_slip:-t_slip, ...])
            losses.append(loss.item())
            loss.backward()
            return loss
        optimizer.step(closure)
    
    if e % 10 == 0:
        with torch.no_grad():
            out = seq(inputs[:100, ...])
            loss = criterion(out, targets[:100, ...])
            test_out = seq(test_inputs)
            test_loss = criterion(test_out, test_targets)
            test_losses.append(test_loss.item())
        print('Epoch: ', e, ', training loss: ', loss.item(), ', test loss: ', test_loss.item(), sep='')

plt.plot(np.array(losses))
plt.plot(test_losses)
pickle.dump(seq, file=open('output/nn/lstm_5_10_10_10_1.pickle', 'wb'))
pickle.dump(dict(losses=losses, test_losses=test_losses),
            file=open('output/nn/losses_5_10_10_10_1.pickle', 'wb'))<jupyter_output><empty_output><jupyter_text>Evaluate the model<jupyter_code>seq = pickle.load(open('output/nn/lstm_5_10_10_10_1.pickle', 'rb'))
# losses, test_losses = pickle.load(open('output/nn/losses_5_10_10_10_1.pickle', 'rb'))<jupyter_output><empty_output><jupyter_text>Train sample<jupyter_code>with torch.no_grad():
    out = seq(inputs)
out.size()
criterion(out, targets).item()
y_true = targets[:, warmup:, :].numpy()
y_pred = out.unsqueeze(dim=2)[:, warmup:, :].detach().numpy().copy()
y_pred[y_pred > 0.5] = 1
y_pred[y_pred <= 0.5] = 0
confusion_matrix(y_true.flatten(), y_pred.flatten()) / len(y_true.flatten())
j = np.random.randint(0, N_samples)
y_hat = out.detach().numpy()

# y_hat_df = DataFrame(data=y_hat[j,...], columns='Trend')
res_j = paths[j].assign(Trend_hat=y_hat[j,...])
res_j[['Close', 'Trend', 'Trend_hat']].plot(figsize=(12, 5))
plt.axhline(0.5, c='grey', alpha=0.5);<jupyter_output><empty_output><jupyter_text>Test sample<jupyter_code>with torch.no_grad():
    test_out = seq(test_inputs)
test_targets.size(), test_out.size()
criterion(test_out, test_targets).item()
y_true = test_targets[:, warmup:, :].numpy()
y_pred = test_out[:, warmup:, :].detach().numpy().copy()
y_pred[y_pred > 0.5] = 1
y_pred[y_pred <= 0.5] = 0
confusion_matrix(y_true.flatten(), y_pred.flatten()) / len(y_true.flatten())
j = np.random.randint(0, test_samples)
y_hat = test_out.detach().numpy()

# y_hat_df = DataFrame(data=y_hat[j,...], columns='Trend')
res_j = test_paths[j].assign(Trend_hat=y_hat[j,...])
res_j[['Close', 'Trend', 'Trend_hat']].plot(figsize=(12, 5))
plt.axhline(0.5, c='grey', alpha=0.5);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16177727143264028, 'fraction_numerical': 0.033252634577043576, 'mean_word_length': 3.203681532475307, 'pattern_counts': {'":': 0, '<': 106, '<?xml version=': 0, '>': 106, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '592512', 'license': 'MIT', 'n_tokens_mistral': 11211, 'n_tokens_neox': 10828, 'n_words': 1990, 'path': 'Time Series Prediction Example.ipynb', 'repo_name': 'peerchen/trend-following-1', 'size': '28093'}