<jupyter_start><jupyter_text>Stack-augmented RNN![caption](https://usercontent1.hubstatic.com/6172838_f260.jpg) This notebook reproduces experiment from article http://arxiv.org/abs/1503.01007  Task and architecture__The task__ is to train an RNN to generate sequences from expression `|`$ a^n b^m c^{n+m} $ * n and m are integers * Performance metrics:  * Correct format - `|`, some a, some b, some c     * ||aaacbbcba would be an example of incorrect one  * Number of C letters to be as close to sum of A band B letters as possible     * Ideally, c = a + b     Model we use: * Stack-augmented RNN Trained as a language model (predicting next symbols given preceding history)  Sequence generator<jupyter_code>def generate_sequence(batch_size = 10,crop_length = 100 ):
    """
    Generates sequence from pattern [0  1*n 2*m 3*(n+m)]
    """
    sequences=[]
    for i in range(batch_size):
        seq = [0]
        while len(seq) < crop_length:
            n,m = np.random.randint(1,10,2)
        
            seq += [0]+ [1]*n+[2]*m+[3]*(n+m)
        seq = seq[:crop_length]
        sequences.append(seq)
    return np.array(sequences,dtype='int32')

alphabet = np.array(list('|abc'))
#quality evaluation
from operator import add
import re
def get_metrics(gen_sequences,alphabet):    
    strings = map(lambda v:reduce(add,v), map(alphabet.__getitem__,gen_sequences))
    
    #at least one complete string
    strings = filter(lambda v: '|' in v,strings)
    
    #cut off last unfinished string, if any
    
    strings = map(lambda v: v[:-1-v[::-1].index('|')],strings)
    
    all_strings = '|'.join(strings)
    
    
    
    seqs = filter(len,all_strings.split('|'))
    
    matches = map(lambda seq:re.match(r"^a+b+c+",seq),seqs)
    
    is_correct = map( lambda seq,m: m is not None and m.pos==0 and m.endpos == len(seq), 
                        seqs, matches)
    
    
    
    
    correct_seqs = np.array(seqs)[np.array(is_correct,dtype='bool')]
    
    seqs_error = map(lambda s: s.count('a')+ s.count('b') - s.count('c'),correct_seqs)
    
    return np.mean(is_correct), np.mean(np.abs(seqs_error))<jupyter_output><empty_output><jupyter_text>samples<jupyter_code>%%time
map(''.join,map(alphabet.__getitem__,generate_sequence(5,100)))<jupyter_output>CPU times: user 4 ms, sys: 0 ns, total: 4 ms
Wall time: 1.42 ms<jupyter_text>Constants<jupyter_code># length of training/generation sequence
SEQ_LENGTH = 50

# Sequences in a minibatch
BATCH_SIZE = 100

# Total epochs
N_EPOCHS = 5000

#how often to print metrics
REPORT_RATE = 100
#Reference sequences for training
sequences_batch = T.matrix(dtype="int32",name="reference_sequences")

#Batch size (theano expression)
batch_size = sequences_batch.shape[0]<jupyter_output><empty_output><jupyter_text>Define RNNA "simple" scheme of what was there in the paper, followed exactly by the code below* Time goes from left to right* Layers start at the bottom and go up* memory states are green, memory updates are purple* inputs and outputs are blue traingles* transformations are in pale yellow![canvas](http://s32.postimg.org/vefrk7vqt/stack_gru.png)<jupyter_code>import lasagne
from lasagne.layers import DenseLayer, ConcatLayer, InputLayer, EmbeddingLayer

import agentnet
from agentnet.resolver import ProbabilisticResolver

from agentnet.memory import GRUMemoryLayer, StackAugmentation


from agentnet.agent import Recurrence



#input letter goes here
output_shape = (None,)
observation_layer = InputLayer(output_shape,name="obs_input")


# embedding the input layer
n_tokens = len(alphabet)
obs_embedding = EmbeddingLayer(observation_layer,
                                              input_size=n_tokens,
                                              output_size=n_tokens,
                                              name = "input_embedding")




#GRU n units
n_hid_1 = 32

#previous GRU state goes here
prev_gru_layer = InputLayer((None,n_hid_1),name="prev_rnn_state")


# previous stack goes here
stack_width = 3
stack_depth = 50
prev_stack_layer = InputLayer((None,stack_depth,stack_width))






# Stack controls - push, pop and no-op
stack_controls_layer = lasagne.layers.DenseLayer(prev_gru_layer,3,
                                                nonlinearity= lasagne.nonlinearities.softmax,
                                                name = "stack controls")

# stack input
stack_input_layer = lasagne.layers.DenseLayer(prev_gru_layer,stack_width,
                                             nonlinearity = lasagne.nonlinearities.tanh,
                                             name = "stack input")
    
    
#new stack state  
next_stack = StackAugmentation(stack_input_layer,
                              prev_stack_layer,
                              stack_controls_layer)


#stack top (used for RNN)
stack_top = lasagne.layers.SliceLayer(next_stack,0,1)



#new GRU state
gru = GRUMemoryLayer(n_hid_1,
                     ConcatLayer([obs_embedding,stack_top]),
                     prev_gru_layer)





##Outputs

# next letter probabilities

probability_layer = lasagne.layers.DenseLayer(gru,
                                         num_units = n_tokens,
                                         nonlinearity=  lasagne.nonlinearities.softmax,
                                         name="policy_original")

#resolver picks a particular letter in generation mode

resolver = ProbabilisticResolver(probability_layer,
                                assume_normalized=True,
                                name="resolver")


#verify that letter shape matches
assert tuple(lasagne.layers.get_output_shape(resolver)) == tuple(output_shape)


#define a dictionary that maps agent's "next memory" output to its previous memory input
from collections import OrderedDict
memory_dict = OrderedDict([
            (gru,prev_gru_layer),
            (next_stack, prev_stack_layer),
    ])<jupyter_output><empty_output><jupyter_text>Training mode* Here we define an in-training version of out generator.* To speed up training, we will feed agent with a reference sequences.* So, agent pretends to generate letters, but actually is fed with the reference ones.<jupyter_code>#define an input layer that stores sequences
sequences_input_layer = InputLayer((None,SEQ_LENGTH),
                                   input_var= sequences_batch,
                                   name = "reference sequences"
                                  )




#and another one that only pretends to, while actually uses the reference letters
# we will use it for training.
generator_passive = Recurrence(
    
    #we use out previously defined dictionary to update recurrent network parameters
    state_variables = memory_dict,
    
    #we feed in reference sequence into "prev letter" input layer, tick by tick along the axis=1.
    input_sequences = {observation_layer:sequences_input_layer},
    
    #we track agent would-be actions and probabilities 
    tracked_outputs = [resolver,probability_layer],
    
    n_steps = SEQ_LENGTH,
    
    #finally, we define an optional batch size param 
    #(if omitted, it will be inferred from inputs or initial value providers if there are any)
    batch_size = batch_size,
    
    )<jupyter_output><empty_output><jupyter_text>Unroll the recurrence* returns sessions of shape [batch_size, sequence_length, variable_size] for each variable* this is still happening inside a lasagne networks  * which allows you to, for example, create a recurrence inside another recurrence<jupyter_code>#get lasagne layers for sequences

#sequence of predicted probabilities
probas_seq_layer = generator_passive[probability_layer]

# gru state sequence (can extract stack likewise)
gru_seq_layer = generator_passive[gru]



# for this task we only need probabilities :)
# probas_seq_layer

probas_seq = lasagne.layers.get_output(probas_seq_layer)<jupyter_output><empty_output><jupyter_text>The rest is a regular lasagne  Weights<jupyter_code>#Obtain them
weights = lasagne.layers.get_all_params(probas_seq_layer,
                                        trainable=True,
                                       )
weights
total_weights = int(T.sum([T.prod(w.shape) for w in weights]).eval())
print "Total weights:", total_weights<jupyter_output>Total weights: 4186<jupyter_text>Loss functionsimple crossentropy<jupyter_code># take all predicitons but for last one (we don't have it's 'next')
predicted_probas = probas_seq[:,:-1]

# crop probabilities to avoid -Inf logarithms
predicted_probas = T.maximum(predicted_probas,1e-10)

# correct answers
references = sequences_batch[:,1:]



# familiar lasagne crossentropy
model_loss = lasagne.objectives.categorical_crossentropy(
    predicted_probas.reshape([-1,n_tokens]),
    references.ravel()
).mean()




#Regularizer for kicks
from lasagne.regularization import regularize_network_params, l2
reg_l2 = regularize_network_params(resolver,l2)*10**-5
loss = model_loss + reg_l2

updates = lasagne.updates.adadelta(loss,weights,learning_rate=0.1)<jupyter_output><empty_output><jupyter_text>Compile the training and evaluation steps* Since we unroll scans, the first compile may take approx. 3.5 cups of tea* If you want it to just compile something, consider either of the two: * setting unroll_scan=False in recurrence and adding generator_passive.get_automatic_updates() * reducing sequence length to, say, 20<jupyter_code>train_fun = theano.function([sequences_batch],[loss],updates=updates)
    
evaluation_fun = theano.function([sequences_batch],[loss,model_loss,reg_l2])<jupyter_output><empty_output><jupyter_text>Generating letters recurrently* In this mode, generator's i+1'st input is it's i-th output* the only difference in terms of code is that instead of reading letters from sequence,  * we feed bach the previously generated ones<jupyter_code># Recurrent state dictionary: maps state outputs to prev_state inputs
feedback_dict = OrderedDict([
            (gru,prev_gru_layer),
            (next_stack, prev_stack_layer),
            (resolver,observation_layer) #here we feed actions back as next observations
    ])


# generation batch size
gen_batch_size = T.scalar('generated batch size','int32')



#define a lasagne recurrence that actually generates letters
# it will be used to generate sample sequences.

generator_active = Recurrence(
    state_variables = feedback_dict,
    tracked_outputs = [resolver,probability_layer],
    
    #note that now we do need to provide batch size since generator does not have any inputs
    batch_size = gen_batch_size,
    n_steps = SEQ_LENGTH
    )
#get sequences as before
generated_probas_seq,generated_action_seq = lasagne.layers.get_output([generator_active[probability_layer],
                                                                      generator_active[resolver]])


# ~0.5 tea cups
get_sequences = theano.function([gen_batch_size],generated_action_seq )
#sample output
map(alphabet.__getitem__,get_sequences(3))<jupyter_output><empty_output><jupyter_text>Training loop:* Generating sequences* Training on them* Printing metrics every once in a while<jupyter_code>from collections import defaultdict

metrics = defaultdict(dict)

for i in range(N_EPOCHS):
    
    #generate reference
    new_batch = generate_sequence(BATCH_SIZE,SEQ_LENGTH)
    #feed the monster
    train_fun(new_batch)
    
    #Display metrics once in a while
    if i % REPORT_RATE==0:
        
        loss_components = evaluation_fun(new_batch)
        print "iter:%i\tfull:%.5f\tllh:%.5f\treg:%.5f"%tuple([i]+map(float,loss_components))        
        
        metrics['crossentropy'][i]=float(loss_components[1])
        

        examples = get_sequences(1000)
        
        correctness_ratio,c_count_mae = get_metrics(examples,alphabet)
        

        metrics["correctness_rates"][i] = correctness_ratio
        metrics["c_count_errors"][i]=c_count_mae
        
        print "Correct sequences ratio: %.5f"%(correctness_ratio)
        print "C-s count mean abs error: %.5f"%(c_count_mae)
        
        for tid_line in examples[:3]:
            print ' '.join(map(alphabet.__getitem__,tid_line))

    
for metric in metrics:
    plt.figure(figsize=[14,8])
    plt.plot(*zip(*sorted(metrics[metric].items(),key=lambda (k,v):k)),label=metric)
    plt.legend(loc='best')
    plt.ylabel("popugai")
    plt.xlabel("epoch")
    plt.grid()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08788368336025848, 'fraction_numerical': 0.010339256865912763, 'mean_word_length': 3.3079331941544887, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 43, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 27, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '68533', 'license': 'MIT', 'n_tokens_mistral': 3775, 'n_tokens_neox': 3600, 'n_words': 975, 'path': 'examples/Stack RNN for formal sequence modelling.ipynb', 'repo_name': '17minutes/AgentNet', 'size': '12380'}