<jupyter_start><jupyter_text>sentences = [  "How are you",  "Who are you",  "Who are they",  "Who are we",  "Who am I",  "Who am I",  "Where are you going"] replace to training dataset from training datas 1. all the elements in sentences are integers, elements equal to words in this model 2. read all elements from training data with randomized segments having same orders<jupyter_code>path1 = "C:/Users/gzkei/PycharmProjects/cis667-secretary-problem/data/first_training_set.csv"
path2 = "C:/Users/gzkei/PycharmProjects/cis667-secretary-problem/data/second_training_set.csv"
path3 = "C:/Users/gzkei/PycharmProjects/cis667-secretary-problem//data/third_training_set.csv"
path4 = "C:/Users/gzkei/PycharmProjects/cis667-secretary-problem//data/fourth_training_set.csv"
path5 = "C:/Users/gzkei/PycharmProjects/cis667-secretary-problem//data/fifth_training_set.csv"


# pick 5000 (each 1000, imply by the parameter inside) segments in each dataset with randomized segments

sentences = []

import number_helpers

sen1 = number_helpers.augment_sentences(path1, 10, 100)
sen2 = number_helpers.augment_sentences(path2, 10, 100)
sen3 = number_helpers.augment_sentences(path3, 10, 100)
sen4 = number_helpers.augment_sentences(path4, 10, 100)
sen5 = number_helpers.augment_sentences(path5, 10, 100)

sentences.extend(sen1)
sentences.extend(sen2)
sentences.extend(sen3)
sentences.extend(sen4)
sentences.extend(sen5)

print(sentences[:5])
print('------------------------------------------------------------------------------------------------------------------------')
print('length:  '+str(len(sentences)))<jupyter_output>[['3', '0', '3', '0', '2', '3', '2', '2', '3', '2', '0', '3', '0', '4', '2', '2', '0', '0', '2', '7', '1', '6', '2', '0', '1', '0', '3', '1', '1', '1', '4', '2', '4', '0', '5', '1', '2', '8', '0', '4', '1', '0', '3', '7', '0', '2', '4', '0', '3', '3', '1', '1', '6', '4', '1', '1', '0', '0', '2', '1', '4', '4', '2', '0', '0', '3', '0', '3', '3', '3', '0', '3', '5', '1', '3', '1', '4', '5', '3', '2', '0', '3', '5', '0', '0', '2', '1', '0', '5', '2', '0', '2', '0', '7', '0', '3', '2', '2', '6', '2', '-1'], ['0', '0', '3', '2', '3', '4', '3', '0', '2', '0', '2', '4', '2', '0', '1', '1', '4', '2', '1', '1', '3', '4', '1', '0', '5', '1', '1', '1', '2', '0', '3', '0', '0', '1', '5', '1', '0', '3', '2', '2', '5', '2', '1', '6', '7', '1', '0', '5', '6', '4', '3', '4', '0', '0', '3', '0', '4', '1', '0', '7', '5', '1', '1', '0', '1', '3', '8', '0', '1', '4', '1', '5', '4', '3', '4', '4', '0', '6', '4', '3', '0', '3', '1', '0', '2', '3', '1', '7', '0', '0', '3', '3', '0', '0', '0', '7', '7', '2', [...]<jupyter_text>start to set building<jupyter_code># Make a dictionary mapping each word to a one-hot tensor
words = set()
for sentence in sentences:
    for word in sentence:
        #for word in sentence.split(" "):
        words.add(word)
words = tuple(words)  # deterministic order

print(words[:5])

# PyTorch LSTM expects 3d tensors representing (sequence length, batch size, number of features)
I = tr.eye(len(words))


# 将数据放入gpu
I = I.to(device) 
I = I.cuda()

dictionary = {
    word: I[w].cuda().reshape(1, 1, len(words))
    for w, word in enumerate(words)}

print(len(dictionary))<jupyter_output>('3', '4', '5', '0', '8')
11<jupyter_text>try to train the LSTM<jupyter_code># Define a small LSTM recurrent neural network with linear hidden-to-output layer
class Net(tr.nn.Module):
    def __init__(self, hidden_size):
        super(Net, self).__init__()
        self.lstm = tr.nn.LSTM(input_size=len(words), hidden_size=hidden_size)
        self.readout = tr.nn.Linear(in_features=hidden_size, out_features=len(words))
        self.lstm.to(device)
        self.readout.to(device)

    def forward(self, x, v=None):
        _, v = self.lstm(x) if v is None else self.lstm(x, v)  # update hidden from input
        h, c = v  # LSTM hidden vector and internal so-called "cell state"
        y = self.readout(h)  # get output from hidden
        y = tr.softmax(y, dim=-1)  # make sure output is a probability distribution
        return y, v


print(Net(3))<jupyter_output>Net(
  (lstm): LSTM(11, 3)
  (readout): Linear(in_features=3, out_features=11, bias=True)
)<jupyter_text>next<jupyter_code>net = Net(3)
# syntax 1
net.to(device)
net.cuda()
opt = tr.optim.SGD(net.parameters(), lr=0.001)
#opt = tr.optim.Adam(net.parameters(), lr=0.1)

for epoch in range(2000):

    batch_loss = 0.

    for sentence in sentences:
        #tokens = sentence.split(" ")
        tokens = sentence
        v = None  # no hidden activation at first time-step
        for t in range(len(tokens) - 1):
            y, v = net(dictionary[tokens[t]], v)
            y_target = dictionary[tokens[t + 1]]

            loss = tr.sum((y - y_target)**2) # MSE
            #loss = -tr.sum(y_target * tr.log(y))  # Cross-entropy
            batch_loss += loss

    batch_loss.backward()
    opt.step()
    opt.zero_grad()

    print(epoch, batch_loss.item())
    # if epoch % 100 == 0: print(epoch, batch_loss.item())<jupyter_output>0 4562.1787109375
1 4484.845703125
2 4422.458984375
3 4371.78369140625
4 4331.923828125
5 4300.955078125
6 4276.25537109375
7 4255.92578125
8 4239.154296875
9 4225.75048828125
10 4215.53076171875
11 4207.9931640625
12 4202.533203125
13 4198.48779296875
14 4195.37890625
15 4192.90869140625
16 4190.85107421875
17 4189.11083984375
18 4187.6123046875
19 4186.30712890625
20 4185.154296875
21 4184.1357421875
22 4183.21826171875
23 4182.3974609375
24 4181.63037109375
25 4180.9541015625
26 4180.32763671875
27 4179.7509765625
28 4179.2265625
29 4178.7333984375
30 4178.27978515625
31 4177.85302734375
32 4177.4619140625
33 4177.09228515625
34 4176.740234375
35 4176.42724609375
36 4176.11865234375
37 4175.8310546875
38 4175.5517578125
39 4175.29833984375
40 4175.05029296875
41 4174.83251953125
42 4174.611328125
43 4174.3955078125
44 4174.19775390625
45 4174.009765625
46 4173.8369140625
47 4173.66015625
48 4173.49462890625
49 4173.33154296875
50 4173.19189453125
51 4173.04736328125
52 4172.90673828[...]<jupyter_text>complete epoch and try prediction<jupyter_code># Try predicting
word = "3"
v = None
print(word)

for t in range(3):
  x = dictionary[word]
  y, v = net(dictionary[tokens[t]], v)
  y = y.squeeze() # ignore singleton dimensions for time-step/example
  w = y.argmax()
  word = words[w]
  prob = y[w]
  print(word, prob.item())
# real prediction
current_sentence = ['3','5','7','4','3','2']
v = None
print(current_sentence)
for c in current_sentence:
    x = dictionary[c]
    
    print('x={x}'.format(x=x))
    print('----------------------------------------')

    y, v = net(dictionary[c], v)

    y = y.squeeze() # ignore singleton dimensions for time-step/example

    w = y.argmax()

    print('y={y}, v={v}, w={w}'.format(y=y,v=v,w=w))
    print('----------------------------------------')

    word = words[w]
    print('word={word} + w={w} ++ {words}'.format(word=word,w=w,words=words))
    print('----------------------------------------')
    prob = y[w]
    print(word, prob.item())
    print('----------------------------------------')
import pickle
with open('beanlstm20211217.pkl', 'wb') as handle:
    pickle.dump(net, handle)  # like_people is a list with data
    input('Finish dumping')<jupyter_output>Finish dumping
{'fraction_non_alphanumeric': 0.22640472592389063, 'fraction_numerical': 0.16032422035993954, 'mean_word_length': 3.393482196741098, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '236260', 'license': 'MIT', 'n_tokens_mistral': 3497, 'n_tokens_neox': 2867, 'n_words': 737, 'path': 'algorithms/BeamLstmTraining-Copy1.ipynb', 'repo_name': 'keith-leung/cis667-secretary-problem', 'size': '7279'}