<jupyter_start><jupyter_text>Adder & Subtractor Practice  Outline* 1. [Coding](coding)* 2. [Run & Analysis](analysis)    * 2.1 [Adder (and test more number of digits)](adder)    * 2.2 [Subtractor](subtractor)    * 2.3 [Combine (and test more data, epoch)](combine)    * 2.4 [Multiplicator (and test binary input)](multiplicator)* 3. [Conclusion](conclusion) ---  1. Coding<jupyter_code>import numpy as np
from keras import layers
from keras.models import Sequential
from six.moves import range
class CharacterTable(object):
    def __init__(self, chars):
        self.chars = sorted(set(chars))
        self.char_indices = dict((c, i) for i, c in enumerate(self.chars))
        self.indices_char = dict((i, c) for i, c in enumerate(self.chars))
    
    def encode(self, C, num_rows):
        x = np.zeros((num_rows, len(self.chars)))
        for i, c in enumerate(C):
            x[i, self.char_indices[c]] = 1
        return x
    
    def decode(self, x, calc_argmax=True):
        if calc_argmax:
            x = x.argmax(axis=-1)
        return ''.join(self.indices_char[x] for x in x)
class colors:
    ok = '\033[92m'
    fail = '\033[91m'
    close = '\033[0m'
def vectorize(ctable, questions, expected, INPUT_MAXLEN, OUTPUT_MAXLEN, chars):
    x = np.zeros((len(questions), INPUT_MAXLEN, len(chars)), dtype=np.bool)
    y = np.zeros((len(questions), OUTPUT_MAXLEN, len(chars)), dtype=np.bool)
    for i, sentence in enumerate(questions):
        x[i] = ctable.encode(sentence, INPUT_MAXLEN)
    for i, sentence in enumerate(expected):
        y[i] = ctable.encode(sentence, OUTPUT_MAXLEN)
    
    indices = np.arange(len(y))
    np.random.shuffle(indices)
    x = x[indices]
    y = y[indices]
    
    split_at = len(x) - len(x) // 10
    (x_train, x_val) = x[:split_at], x[split_at:]
    (y_train, y_val) = y[:split_at], y[split_at:]
    
    print('Training Data:')
    print(x_train.shape)
    print(y_train.shape)
    
    print('Validation Data:')
    print(x_val.shape)
    print(y_val.shape)
    
    return x_train, y_train, x_val, y_val
def build_model(LAYERS, HIDDEN_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, chars):
    model = Sequential()
    model.add(layers.LSTM(HIDDEN_SIZE, input_shape=(INPUT_MAXLEN, len(chars))))
    model.add(layers.RepeatVector(OUTPUT_MAXLEN))
    for _ in range(LAYERS):
        model.add(layers.LSTM(HIDDEN_SIZE, return_sequences=True))
    
    model.add(layers.TimeDistributed(layers.Dense(len(chars), activation='softmax')))
    model.compile(loss='categorical_crossentropy',
                  optimizer='adam',
                  metrics=['accuracy'])
    model.summary()
    
    return model
def training(model, ctable, BATCH_SIZE, ITERATION, x_train, y_train, x_val, y_val, REVERSE, test=True):
    for iter in range(ITERATION):
        print()
        print('-' * 50)
        print('Iteration', iter + 1)
        model.fit(x_train, y_train,
                  batch_size=BATCH_SIZE,
                  epochs=1,
                  validation_data=(x_val, y_val),
                  verbose=2)
        
        if test:
            testing(model, ctable, x_val, y_val, REVERSE)
def testing(model, ctable, x_val, y_val, REVERSE):
    for i in range(10):
        ind = np.random.randint(0, len(x_val))
        rowx, rowy = x_val[np.array([ind])], y_val[np.array([ind])]
        preds = model.predict_classes(rowx, verbose=0)
        q = ctable.decode(rowx[0])
        correct = ctable.decode(rowy[0])
        guess = ctable.decode(preds[0], calc_argmax=False)
        print('Q', q[::-1] if REVERSE else q, end=' ')
        print('T', correct, end=' ')
        if correct == guess:
            print(colors.ok + '☑' + colors.close, end=' ')
        else:
            print(colors.fail + '☒' + colors.close, end=' ')
        print(guess)<jupyter_output><empty_output><jupyter_text>2. Run & Analysis  2.1 Adder  2.1.1 Parameters<jupyter_code>TRAINING_SIZE = 50000
DIGITS = 3
INPUT_MAXLEN = DIGITS + 1 + DIGITS
OUTPUT_MAXLEN = DIGITS + 1
REVERSE = True
    
chars = '0123456789+ '
ctable = CharacterTable(chars)
    
HIDDEN_SIZE = 128
LAYERS = 1
    
ITERATION = 50
BATCH_SIZE = 128<jupyter_output><empty_output><jupyter_text>2.1.2 Generate data<jupyter_code>def gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS, REVERSE, chars):
    questions = []
    expected = []
    seen = set()
    
    while len(questions) < TRAINING_SIZE:
        fn = lambda: int(''.join(np.random.choice(list('0123456789')) for _ in range(np.random.randint(1, DIGITS + 1))))
        a, b = fn(), fn()
        
        key = tuple(sorted((a, b)))
        if key in seen:
            continue
        seen.add(key)
        
        q = '{}+{}'.format(a, b)
        query = q + ' ' * (INPUT_MAXLEN - len(q))
        
        ans = str(a + b)
        ans += ' ' * (OUTPUT_MAXLEN - len(ans))
        
        if REVERSE:
            query = query[::-1]
        
        questions.append(query)
        expected.append(ans)
    print('Total questions:', len(questions))
    
    return vectorize(ctable, questions, expected, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)<jupyter_output><empty_output><jupyter_text>2.1.3 Training<jupyter_code>x_train, y_train, x_val, y_val = gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS,REVERSE, chars)
model = build_model(LAYERS, HIDDEN_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)
training(model, ctable, BATCH_SIZE, ITERATION, x_train, y_train, x_val, y_val, REVERSE)<jupyter_output>Total questions: 50000
Training Data:
(45000, 7, 12)
(45000, 4, 12)
Validation Data:
(5000, 7, 12)
(5000, 4, 12)
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm_1 (LSTM)                (None, 128)               72192     
_________________________________________________________________
repeat_vector_1 (RepeatVecto (None, 4, 128)            0         
_________________________________________________________________
lstm_2 (LSTM)                (None, 4, 128)            131584    
_________________________________________________________________
time_distributed_1 (TimeDist (None, 4, 12)             1548      
=================================================================
Total params: 205,324
Trainable params: 205,324
Non-trainable params: 0
_________________________________________________________________

----------------------[...]<jupyter_text>2.1.4 Analysis - 在Epoch 22時達到99%的精準度，Epoch 22達到99.9%- 三位數相加對模型算是非常簡單的問題  2.1.5 Test different number of digits and analysis<jupyter_code>DIGITS = 10
INPUT_MAXLEN = DIGITS + 1 + DIGITS
OUTPUT_MAXLEN = DIGITS + 1

x_train, y_train, x_val, y_val = gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS,REVERSE, chars)
model = build_model(LAYERS, HIDDEN_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)
training(model, ctable, BATCH_SIZE, ITERATION, x_train, y_train, x_val, y_val, REVERSE)<jupyter_output>Total questions: 50000
Training Data:
(45000, 21, 12)
(45000, 11, 12)
Validation Data:
(5000, 21, 12)
(5000, 11, 12)
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm_5 (LSTM)                (None, 128)               72192     
_________________________________________________________________
repeat_vector_3 (RepeatVecto (None, 11, 128)           0         
_________________________________________________________________
lstm_6 (LSTM)                (None, 11, 128)           131584    
_________________________________________________________________
time_distributed_3 (TimeDist (None, 11, 12)            1548      
=================================================================
Total params: 205,324
Trainable params: 205,324
Non-trainable params: 0
_________________________________________________________________

------------------[...]<jupyter_text>- 精準度只到72%左右- 越多位數需要更多的training data和epoch才能學得好- 如果需要做更多位數的運算，應該像硬體設計的方式那樣，先設計1 bit的full adder後串聯起來 ---  2.2 Subtractor  2.2.1 Parameters<jupyter_code>TRAINING_SIZE = 50000
DIGITS = 3
INPUT_MAXLEN = DIGITS + 1 + DIGITS
OUTPUT_MAXLEN = DIGITS + 1
REVERSE = True
    
chars = '0123456789- '
ctable = CharacterTable(chars)
    
HIDDEN_SIZE = 128
LAYERS = 1
    
ITERATION = 50
BATCH_SIZE = 128<jupyter_output><empty_output><jupyter_text>2.2.2 Generate data<jupyter_code>def gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS, REVERSE, chars):
    questions = []
    expected = []
    seen = set()
    
    while len(questions) < TRAINING_SIZE:
        fn = lambda: int(''.join(np.random.choice(list('0123456789')) for _ in range(np.random.randint(1, DIGITS + 1))))
        a, b = fn(), fn()
        
        key = tuple(sorted((a, b)))
        if key in seen or a < b:
            continue
        seen.add(key)
        
        q = '{}-{}'.format(a, b)
        query = q + ' ' * (INPUT_MAXLEN - len(q))
        
        ans = str(a - b)
        ans += ' ' * (OUTPUT_MAXLEN - len(ans))
        
        if REVERSE:
            query = query[::-1]
        
        questions.append(query)
        expected.append(ans)
    print('Total questions:', len(questions))
    
    return vectorize(ctable, questions, expected, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)<jupyter_output><empty_output><jupyter_text>2.2.3 Training<jupyter_code>x_train, y_train, x_val, y_val = gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS,REVERSE, chars)
model = build_model(LAYERS, HIDDEN_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)
training(model, ctable, BATCH_SIZE, ITERATION, x_train, y_train, x_val, y_val, REVERSE)<jupyter_output>Total questions: 50000
Training Data:
(45000, 7, 12)
(45000, 4, 12)
Validation Data:
(5000, 7, 12)
(5000, 4, 12)
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm_3 (LSTM)                (None, 128)               72192     
_________________________________________________________________
repeat_vector_2 (RepeatVecto (None, 4, 128)            0         
_________________________________________________________________
lstm_4 (LSTM)                (None, 4, 128)            131584    
_________________________________________________________________
time_distributed_2 (TimeDist (None, 4, 12)             1548      
=================================================================
Total params: 205,324
Trainable params: 205,324
Non-trainable params: 0
_________________________________________________________________

----------------------[...]<jupyter_text>2.2.4 Analysis - Epoch 22時達到99%的精準度- 三位數相減相比相加需要更多時間訓練，但仍然是較簡單的問題 ---  2.3 Combine  2.3.1 Parameters<jupyter_code>TRAINING_SIZE = 50000
DIGITS = 3
INPUT_MAXLEN = DIGITS + 1 + DIGITS + 1 + DIGITS
OUTPUT_MAXLEN = DIGITS + 1
REVERSE = True
    
chars = '0123456789+- '
ctable = CharacterTable(chars)
    
HIDDEN_SIZE = 128
LAYERS = 1
    
ITERATION = 100
BATCH_SIZE = 128<jupyter_output><empty_output><jupyter_text>2.3.2 Generate data<jupyter_code>def gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS, REVERSE, chars):
    questions = []
    expected = []
    seen = set()
    
    while len(questions) < TRAINING_SIZE:
        fn = lambda: int(''.join(np.random.choice(list('0123456789')) for _ in range(np.random.randint(1, DIGITS + 1))))
        fs = lambda: ''.join(np.random.choice(list('+-')))
        
        a, b, c = fn(), fn(), fn()
        sign1, sign2 = fs(), fs()
        
        key = tuple(sorted((a, b, c)))
        ans = a + b if sign1 == '+' else a - b
        ans = ans + c if sign2 == '+' else ans - c
        if key in seen or ans < 0:
            continue
        seen.add(key)

        q = '{}{}{}{}{}'.format(a, sign1, b, sign2, c)
        query = q + ' ' * (INPUT_MAXLEN - len(q))
        
        ans = str(ans)
        ans += ' ' * (OUTPUT_MAXLEN - len(ans))
        
        if REVERSE:
            query = query[::-1]
        
        questions.append(query)
        expected.append(ans)
    print('Total questions:', len(questions))
    
    return vectorize(ctable, questions, expected, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)<jupyter_output><empty_output><jupyter_text>2.3.3 Training<jupyter_code>x_train, y_train, x_val, y_val = gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS,REVERSE, chars)
model = build_model(LAYERS, HIDDEN_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)
training(model, ctable, BATCH_SIZE, ITERATION, x_train, y_train, x_val, y_val, REVERSE)<jupyter_output>Total questions: 50000
Training Data:
(45000, 11, 13)
(45000, 4, 13)
Validation Data:
(5000, 11, 13)
(5000, 4, 13)
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm_7 (LSTM)                (None, 128)               72704     
_________________________________________________________________
repeat_vector_4 (RepeatVecto (None, 4, 128)            0         
_________________________________________________________________
lstm_8 (LSTM)                (None, 4, 128)            131584    
_________________________________________________________________
time_distributed_4 (TimeDist (None, 4, 13)             1677      
=================================================================
Total params: 205,965
Trainable params: 205,965
Non-trainable params: 0
_________________________________________________________________

--------------------[...]<jupyter_text>2.3.4 Analysis - 精準度最高只到72%- 相比兩個數字相加或相減，這個問題更困難，需要更多Data和Epoch  2.3.5 Test more data and epoch, and analysis<jupyter_code>TRAINING_SIZE = 100000
ITERATION = 200

x_train, y_train, x_val, y_val = gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS,REVERSE, chars)
model = build_model(LAYERS, HIDDEN_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)
training(model, ctable, BATCH_SIZE, ITERATION, x_train, y_train, x_val, y_val, REVERSE, test=False)
test(model, ctable, x_val, y_val, REVERSE)<jupyter_output>Total questions: 100000
Training Data:
(90000, 11, 13)
(90000, 4, 13)
Validation Data:
(10000, 11, 13)
(10000, 4, 13)
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm_1 (LSTM)                (None, 128)               72704     
_________________________________________________________________
repeat_vector_1 (RepeatVecto (None, 4, 128)            0         
_________________________________________________________________
lstm_2 (LSTM)                (None, 4, 128)            131584    
_________________________________________________________________
time_distributed_1 (TimeDist (None, 4, 13)             1677      
=================================================================
Total params: 205,965
Trainable params: 205,965
Non-trainable params: 0
_________________________________________________________________

-----------------[...]<jupyter_text>- 增加Data與Epoch次數可以提高精準度，從72%提高至79% ---  2.4 Multiplicator  2.4.1 Parameters<jupyter_code>TRAINING_SIZE = 50000
DIGITS = 3
INPUT_MAXLEN = DIGITS + 1 + DIGITS
OUTPUT_MAXLEN = DIGITS + DIGITS
REVERSE = True
    
chars = '0123456789* '
ctable = CharacterTable(chars)
    
HIDDEN_SIZE = 128
LAYERS = 1
    
ITERATION = 100
BATCH_SIZE = 128<jupyter_output><empty_output><jupyter_text>2.4.2 Generate data<jupyter_code>def gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS, REVERSE, chars):
    questions = []
    expected = []
    seen = set()
    
    while len(questions) < TRAINING_SIZE:
        fn = lambda: int(''.join(np.random.choice(list('0123456789')) for _ in range(np.random.randint(1, DIGITS + 1))))
        a, b = fn(), fn()
        
        key = tuple(sorted((a, b)))
        if key in seen:
            continue
        seen.add(key)
        
        q = '{}*{}'.format(a, b)
        query = q + ' ' * (INPUT_MAXLEN - len(q))
        
        ans = str(a * b)
        ans += ' ' * (OUTPUT_MAXLEN - len(ans))
        
        if REVERSE:
            query = query[::-1]
        
        questions.append(query)
        expected.append(ans)
    print('Total questions:', len(questions))
    
    return vectorize(ctable, questions, expected, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)<jupyter_output><empty_output><jupyter_text>2.4.3 Training<jupyter_code>x_train, y_train, x_val, y_val = gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS,REVERSE, chars)
model = build_model(LAYERS, HIDDEN_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)
training(model, ctable, BATCH_SIZE, ITERATION, x_train, y_train, x_val, y_val, REVERSE)<jupyter_output>Total questions: 50000
Training Data:
(45000, 7, 12)
(45000, 6, 12)
Validation Data:
(5000, 7, 12)
(5000, 6, 12)
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm_3 (LSTM)                (None, 128)               72192     
_________________________________________________________________
repeat_vector_2 (RepeatVecto (None, 6, 128)            0         
_________________________________________________________________
lstm_4 (LSTM)                (None, 6, 128)            131584    
_________________________________________________________________
time_distributed_2 (TimeDist (None, 6, 12)             1548      
=================================================================
Total params: 205,324
Trainable params: 205,324
Non-trainable params: 0
_________________________________________________________________

----------------------[...]<jupyter_text>2.4.4 Analysis - 精準度最高只到75%左右- 模型最先學會了任意數乘以0和乘以1的問題- 任意數乘以5的準確度也較高- 到了中期可以部分解決任意數乘以個位數，或者乘以個位數加上0 (如*30,*200) 的例子 - 後期能解決部分任意數乘以任意數的例子，但整體精準度仍偏低  2.4.5 Test binary multiplicator and Analysis<jupyter_code>DIGITS = 10
INPUT_MAXLEN = DIGITS + 1 + DIGITS
OUTPUT_MAXLEN = DIGITS + DIGITS
REVERSE = True
    
chars = '01* '
ctable = CharacterTable(chars)

def gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS, REVERSE, chars):
    questions = []
    expected = []
    seen = set()
    
    while len(questions) < TRAINING_SIZE:
        fn = lambda: ''.join(np.random.choice(list('01')) for _ in range(np.random.randint(1, DIGITS + 1)))
        a, b = int(fn(), 2),  int(fn(), 2)
        
        key = tuple(sorted((a, b)))
        if key in seen:
            continue
        seen.add(key)
        
        q = '{}*{}'.format(bin(a)[2:], bin(b)[2:])
        query = q + ' ' * (INPUT_MAXLEN - len(q))
        
        ans = '{}'.format(bin(a*b)[2:])
        ans += ' ' * (OUTPUT_MAXLEN - len(ans))
        
        if REVERSE:
            query = query[::-1]
        
        questions.append(query)
        expected.append(ans)
    print('Total questions:', len(questions))
    
    return vectorize(ctable, questions, expected, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)

x_train, y_train, x_val, y_val = gen_data(ctable, TRAINING_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, DIGITS,REVERSE, chars)
model = build_model(LAYERS, HIDDEN_SIZE, INPUT_MAXLEN, OUTPUT_MAXLEN, chars)
training(model, ctable, BATCH_SIZE, ITERATION, x_train, y_train, x_val, y_val, REVERSE)<jupyter_output>Total questions: 50000
Training Data:
(45000, 21, 4)
(45000, 20, 4)
Validation Data:
(5000, 21, 4)
(5000, 20, 4)
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
lstm_1 (LSTM)                (None, 128)               68096     
_________________________________________________________________
repeat_vector_1 (RepeatVecto (None, 20, 128)           0         
_________________________________________________________________
lstm_2 (LSTM)                (None, 20, 128)           131584    
_________________________________________________________________
time_distributed_1 (TimeDist (None, 20, 4)             516       
=================================================================
Total params: 200,196
Trainable params: 200,196
Non-trainable params: 0
_________________________________________________________________

----------------------[...]
{'fraction_non_alphanumeric': 0.15624086524408068, 'fraction_numerical': 0.04521095196336354, 'mean_word_length': 3.293453252457645, 'pattern_counts': {'":': 0, '<': 65, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '362583', 'license': 'MIT', 'n_tokens_mistral': 7333, 'n_tokens_neox': 6536, 'n_words': 1414, 'path': 'nlp-adder-subtractor.ipynb', 'repo_name': 'nitsaick/nlp-seq2seq-adder-subtractor', 'size': '20526'}