<jupyter_start><jupyter_text>TF Seq2seq1) For seq2seq to work we need eos and pad tokens. We use pad as  since there are no major repurcussions, and eos is stored in vocab already. Unless,  has a particular embedding. 2)<jupyter_code>import os
os.chdir('../model/')
import tensorflow as tf
import pickle
import numpy as np
from tensorflow.contrib.rnn import LSTMCell, LSTMStateTuple
def get_word_embeddings(filename):
    try:
        with np.load(filename) as data:
            return data["embeddings"]

    except IOError:
        raise MyIOError(filename)
        
def dataset_load(domain_tr_data_path, vocab_path):
    with open(domain_tr_data_path,'r') as p1:
        domain_tr_data = pickle.load(p1)
    with open(vocab_path,'r') as p1:
        vocab = pickle.load(p1)
        
    domain_tr_data = map(lambda x: x[0],domain_tr_data)
    idd_domain_tr_data = map(lambda x: [vocab[word] for word in x], domain_tr_data)
    return idd_domain_tr_data, vocab
domain_name = 'laptop'
domain_tr_data_path = '../data/Final_joint_data_absa//Domains/Laptop/Normal__normal_training_list.pickle'
embeddings_path = '../data/Embeddings/Pruned/np_glove_200d_trimmed.npz'
embeddings_name = 'glove200d'
vocab_path = '../data/vocab_to_id.pkl'

with open(vocab_path,'r') as p1:
        vocab = pickle.load(p1)
word_embeddings_np = get_word_embeddings(embeddings_path)
pad_token = '<PAD>' 
eos_token = '<END>'
PAD = vocab[pad_token]
EOS = vocab[eos_token]
vocab_size = len(vocab)
input_embedding_size = 200
encoder_hidden_units = 50 #100
decoder_hidden_units = encoder_hidden_units*2
encoder_inputs = tf.placeholder(shape=(None, None), dtype=tf.int32, name='encoder_inputs')
encoder_inputs_length = tf.placeholder(shape=(None,), dtype=tf.int32, name='encoder_inputs_length')
decoder_targets = tf.placeholder(shape=(None, None), dtype=tf.int32, name='decoder_targets')
embeddings = tf.Variable(word_embeddings_np, name="word_embeds",dtype=tf.float32, trainable=False)
#encoder_inputs_embedded = tf.nn.embedding_lookup(embeddings, encoder_inputs)
'''This is just to test again'''
with tf.variable_scope('seq2seq_encoder'):
        encoder_inputs_embedded = tf.nn.embedding_lookup(embeddings, word_ids)
        encoder_cell = LSTMCell(encoder_hidden_units)
        ((encoder_fw_outputs,
          encoder_bw_outputs),
         (encoder_fw_final_state,
          encoder_bw_final_state)) = (
            tf.nn.bidirectional_dynamic_rnn(cell_fw=encoder_cell,
                                    cell_bw=encoder_cell,
                                    inputs=encoder_inputs_embedded,
                                    sequence_length= sequence_lengths,
                                    dtype=tf.float32, time_major=True)
            ) 
    
        encoder_outputs = tf.concat((encoder_fw_outputs, encoder_bw_outputs),2)
    

        encoder_final_state_c = tf.concat(
            (encoder_fw_final_state.c, encoder_bw_final_state.c), 1)

        encoder_final_state_h = tf.concat(
            (encoder_fw_final_state.h, encoder_bw_final_state.h), 1)

        encoder_final_state = LSTMStateTuple(
            c=encoder_final_state_c,
            h=encoder_final_state_h
            ) #this is useful later

        encoder_contact_rep = tf.concat([encoder_final_state_c,encoder_final_state_h], 1)

'''
with tf.variable_scope("seq2seq_encoder"):
    '''THIS STUFF NEEDS TO BE EDITED, just copy the encoder part into original code'''
    encoder_cell = LSTMCell(encoder_hidden_units)
    ((encoder_fw_outputs,
          encoder_bw_outputs),
         (encoder_fw_final_state,
          encoder_bw_final_state)) = (
            tf.nn.bidirectional_dynamic_rnn(cell_fw=encoder_cell,
                                    cell_bw=encoder_cell,
                                    inputs=encoder_inputs_embedded,
                                    sequence_length=encoder_inputs_length,
                                    dtype=tf.float32, time_major=True)
            ) #'
    
    encoder_outputs = tf.concat((encoder_fw_outputs, encoder_bw_outputs),2)
    

    encoder_final_state_c = tf.concat(
        (encoder_fw_final_state.c, encoder_bw_final_state.c), 1)

    encoder_final_state_h = tf.concat(
        (encoder_fw_final_state.h, encoder_bw_final_state.h), 1)

    encoder_final_state = LSTMStateTuple(
        c=encoder_final_state_c,
        h=encoder_final_state_h
    ) #this is useful later

    encoder_concat_everything = tf.concat([encoder_final_state_c,encoder_final_state_h], 1)
    '''
decoder_cell = LSTMCell(decoder_hidden_units)
encoder_max_time, batch_size = tf.unstack(tf.shape(encoder_inputs))
decoder_lengths = encoder_inputs_length + 4 #3 additional 
W = tf.Variable(tf.random_uniform([decoder_hidden_units, vocab_size], -1, 1), dtype=tf.float32)
b = tf.Variable(tf.zeros([vocab_size]), dtype=tf.float32)
eos_time_slice = EOS*tf.ones([batch_size], dtype=tf.int32, name='EOS')
pad_time_slice = PAD*tf.ones([batch_size], dtype=tf.int32, name='PAD')

eos_step_embedded = tf.nn.embedding_lookup(embeddings, eos_time_slice)
pad_step_embedded = tf.nn.embedding_lookup(embeddings, pad_time_slice)
def loop_fn_initial():
    initial_elements_finished = (0 >= decoder_lengths)  # all False at the initial step
    initial_input = eos_step_embedded
    initial_cell_state = encoder_final_state
    initial_cell_output = None
    initial_loop_state = None  # we don't need to pass any additional information
    return (initial_elements_finished,
            initial_input,
            initial_cell_state,
            initial_cell_output,
            initial_loop_state)

def loop_fn_transition(time, previous_output, previous_state, previous_loop_state):

    def get_next_input():
        output_logits = tf.add(tf.matmul(previous_output, W), b)
        prediction = tf.argmax(output_logits, axis=1)
        next_input = tf.nn.embedding_lookup(embeddings, prediction)
        return next_input
    
    elements_finished = (time >= decoder_lengths) # this operation produces boolean tensor of [batch_size]
                                                  # defining if corresponding sequence has ended

    finished = tf.reduce_all(elements_finished) # -> boolean scalar
    input = tf.cond(finished, lambda: pad_step_embedded, get_next_input)
    state = previous_state
    output = previous_output
    loop_state = None

    return (elements_finished, 
            input,
            state,
            output,
            loop_state)
def loop_fn(time, previous_output, previous_state, previous_loop_state):
    if previous_state is None:    # time == 0
        assert previous_output is None and previous_state is None
        return loop_fn_initial()
    else:
        return loop_fn_transition(time, previous_output, previous_state, previous_loop_state)

decoder_outputs_ta, decoder_final_state, _ = tf.nn.raw_rnn(decoder_cell, loop_fn)
decoder_outputs = decoder_outputs_ta.stack()
decoder_max_steps, decoder_batch_size, decoder_dim = tf.unstack(tf.shape(decoder_outputs))
decoder_outputs_flat = tf.reshape(decoder_outputs, (-1, decoder_dim))
decoder_logits_flat = tf.add(tf.matmul(decoder_outputs_flat, W), b)
decoder_logits = tf.reshape(decoder_logits_flat, (decoder_max_steps, decoder_batch_size, vocab_size))
decoder_prediction = tf.argmax(decoder_logits, 2)
stepwise_cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
    labels=tf.one_hot(decoder_targets, depth=vocab_size, dtype=tf.float32),
    logits=decoder_logits,
)

loss = tf.reduce_mean(stepwise_cross_entropy)
batch_size = 25
num_epochs = 10

def dataset_load(domain_tr_data_path, vocab_path):
    with open(domain_tr_data_path,'r') as p1:
        domain_tr_data = pickle.load(p1)
    with open(vocab_path,'r') as p1:
        vocab = pickle.load(p1)
        
    domain_tr_data = map(lambda x: x[0],domain_tr_data)
    idd_domain_tr_data = map(lambda x: [vocab[word] for word in x], domain_tr_data)
    return idd_domain_tr_data

def get_holdout_data(idd_domain_tr_data, num=10):
    '''Simple function to check if encoder is functioning properly'''
    return idd_domain_tr_data[:num]

def gen_batch(idd_data, batch_size):
    np.random.shuffle(idd_data)
    rem = len(idd_data)%batch_size
    num_batches = (len(idd_data)/batch_size) 
    if rem>0:
        num_batches = num_batches + 1
    
    
    for i in range(num_batches):
        if(i == num_batches -1 and (not rem==0)):
            yield(idd_data[i*batch_size:])
        else:
            yield(idd_data[i*batch_size:(i+1)*batch_size])
            
def batch_modify(inputs, max_sequence_length=None):
    """
    Args:
        inputs:
            list of sentences (integer lists)
        max_sequence_length:
            integer specifying how large should `max_time` dimension be.
            If None, maximum sequence length would be used
    
    Outputs:
        inputs_time_major:
            input sentences transformed into time-major matrix 
            (shape [max_time, batch_size]) padded with 0s
        sequence_lengths:
            batch-sized list of integers specifying amount of active 
            time steps in each input sequence
    """
    
    sequence_lengths = [len(seq) for seq in inputs]
    batch_size = len(inputs)
    
    if max_sequence_length is None:
        max_sequence_length = max(sequence_lengths)
    
    inputs_batch_major = PAD*np.ones(shape=[batch_size, max_sequence_length], dtype=np.int32) # == PAD
    
    for i, seq in enumerate(inputs):
        for j, element in enumerate(seq):
            inputs_batch_major[i, j] = element

    # [batch_size, max_time] -> [max_time, batch_size]
    inputs_time_major = inputs_batch_major.swapaxes(0, 1)

    return inputs_time_major, sequence_lengths
        
def next_feed(batch):
    encoder_inputs_, encoder_input_lengths_ = batch_modify(batch)
    decoder_targets_, _ = batch_modify(
        [(sequence) + [EOS] + [PAD] * 3 for sequence in batch] #additional 3 spaces
    )
    return {
        encoder_inputs: encoder_inputs_,
        encoder_inputs_length: encoder_input_lengths_,
        decoder_targets: decoder_targets_,
    }

def feed_enc(enc_batch):
    
    encoder_inputs_, encoder_input_lengths_ = batch_modify(enc_batch)
    return {encoder_inputs: encoder_inputs_,
            encoder_inputs_length: encoder_input_lengths_}


def feed_enc_with_bridge(encoder_inputs, encoder_input_lengths_):
    
    return {encoder_inputs: encoder_inputs_,
            encoder_inputs_length: encoder_input_lengths_}
idd_data = dataset_load(domain_tr_data_path,vocab_path)
holdout_data = get_holdout_data(idd_data,10)
embed_type = "Glove"
model_path = '../results/seq2seq/{}_seq2seqmodel_embeds{}_{}d_{}hiddenunits.ckpt'.format(domain_name,embed_type,input_embedding_size,encoder_hidden_units)
encoder_path = '../results/seq2seq/{}_ENC_seq2seqmodel_embeds{}_{}d_{}hiddenunits.ckpt'.format(domain_name,embed_type,input_embedding_size,encoder_hidden_units)
batch_size = 150
lr = None
num_epochs = 1
model_exists_already = False

if (lr is None):
    train_op = tf.train.AdamOptimizer().minimize(loss)
else:
    train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(loss)


if not os.path.exists(model_path):
    os.makedirs(model_path)


graph = tf.get_default_graph()
config = tf.ConfigProto(device_count={'GPU': 0})
saver = tf.train.Saver()
#encoder_saver = tf.train.Saver({"seq2seq_encoder": encoder_concat_everything})

loss_track = []

def train_graph(model_exists_already):
    
    with tf.Session(config=config) as sess:
        
        sess.run(tf.global_variables_initializer())

        print("Initialized session")
        if(model_exists_already):
            print("loading existing model")
            saver.restore(sess, model_path)
        
        for epoch in range(num_epochs):
            print("At epoch: {}".format(epoch))
            iters = 0 
            epoch_loss = 0.
            batch_generator = gen_batch(idd_data, batch_size)
            for batch in batch_generator:
                
                fd = next_feed(batch)
                _, l = sess.run([train_op, loss], fd)
    
                if(iters%33==0):
                    print("At batch: {}".format(iters))
                    print("Batch loss: {}".format(l))
                loss_track.append(l)
                iters+=1
                epoch_loss+=l
            print("Epoch training loss: {}".format(epoch_loss/iters))
            
        f_enc = feed_enc(holdout_data)
        encoder_useful_state = sess.run(encoder_concat_everything, f_enc)
            
        saver.save(sess,model_path)
        print("Saved model at: {}".format(model_path))
        #encoder_useful_state = sess.run(encoder_concat_everything)
    return encoder_useful_state


encoder_out = train_graph(False)
encoder_out
'''Cell to continue training'''
with tf.Session(config=config) as sess:
    #saver = tf.train.import_meta_graph('../results/seq2seq/laptop_seq2seqmodel_embedsGlove_200d_100hiddenunits.meta')
    #saver.restore(sess, tf.train.latest_checkpoint('../results/seq2seq/laptop_seq2seqmodel_embedsGlove_200d_100hiddenunits.data-00000-of-00001'))
    #sess.run()
    saver.restore(sess, model_path)
    batch_generator = gen_batch(idd_data, batch_size)
    for batch in batch_generator:        
        fd = next_feed(batch)
        _, l = sess.run([train_op, loss], fd)
        encoder_useful_state = sess.run(encoder_concat_everything, fd)

'''INPUTS'''
input_word_seq_tensor = tf.convert_to_tensor(np.array([[0,1,2,3,1,2],[9,10,11,12,13,12]]),dtype='int32')
max_seq_length = 6
enc_dims = 50
seq_lengths = tf.convert_to_tensor(np.array([4,5]),dtype='int32') #shape n




'''Create mask based on max seq length'''
np_mask_matrix = np.ones((max_seq_length,max_seq_length)) #num drops=n
a = np.array(range(max_seq_length))
np_mask_matrix[np.arange(len(a)), a] = 0
#np_mask_matrix  = np_mask_matrix!=0
tf_mask_matrix = tf.convert_to_tensor(np_mask_matrix, dtype="bool")
padding = tf.constant([[0,0],[0,1]],dtype='int32')
resultant_tensor = tf.expand_dims(input_word_seq_tensor,0)
#print(resultant_tensor.eval())
tensor_seq_lengths = tf.expand_dims(seq_lengths, 0)
seq_lengths_for_dropped = tf.expand_dims(seq_lengths-tf.ones(shape=seq_lengths.shape[0],dtype="int32"),0)
#print(seq_lengths_for_dropped)

#def condition(x):
 #   x.shape[0]< max_seq_length
#def body(x):
 #   Copy

for drop_index in range(max_seq_length):
    '''Create mask function-> note the drop index changes at each iteration'''
    f = lambda word_seq: tf.boolean_mask(word_seq, tf_mask_matrix[:,drop_index])#,padding,"CONSTANT") 
    '''Apply masking and padding'''
    #temp = tf.pad(tf.map_fn(f, input_word_seq_tensor), padding, "CONSTANT") #We apply mask and then pad result to enable concatenation with original 
    #temp = tf.expand_dims(temp,0)
    resultant_tensor = tf.concat([resultant_tensor,tf.expand_dims(tf.pad(tf.map_fn(f, input_word_seq_tensor), padding, "CONSTANT"),0)],0)
    tensor_seq_lengths = tf.concat([tensor_seq_lengths, seq_lengths_for_dropped],0)
#print(resultant_tensor.eval())
#print(resultant_tensor.shape)
#print(tensor_seq_lengths.eval())
#print(tensor_seq_lengths.shape)
resultant_tensor = tf.reshape(resultant_tensor,[resultant_tensor.shape[0]*resultant_tensor.shape[1],resultant_tensor.shape[2]])
tensor_seq_lengths = tf.reshape(tensor_seq_lengths,[tensor_seq_lengths.shape[0]*tensor_seq_lengths.shape[1],])
#seq2seq_out = get_encoded_rep()
#the seq2seq_out = reshaping of
'''get_encoder_reps returns the batched encoder representations (shape = [(input_max_length+2)*(input_batch_size) , encoder_dims]'''
#x = get_encoder_reps(resultant_tensor,tensor_seq_lengths,dims)
seq2seq_out = tf.reshape(get_encoder_reps(resultant_tensor,tensor_seq_lengths), [max_seq_length+1, input_word_seq_tensor.shape[0], enc_dims])

#Each column corresponds to the vector rep for a sentence
#Each row corresponds to the word representation

#We now reduce this by doing an op(row_0,row_i) for all columns
'''NOTE : Hard coded here with subtract have to replace with op(row_0, row_i) for all rows i'''
seq2seq_out = tf.subtract(seq2seq_out,seq2seq_out[0,:])[1:,:]
#So final output should be max_seq_len*n_batches*dims --> do transpose if needed
seq2seq_out = tf.transpose(seq2seq_out,perm=[1,0,2]) #convert to n_batches*max_seq_len*dims or n_batches*words*dims





#apply function operation that takes first index and performs op(first_index, slice)
#print(seq2seq_in.eval())
#print(seq2seq_in.shape)
#print(seq2seq_in_lengths.eval())
#print(seq2seq_in_lengths.shape)

with tf.Session(config=config) as sess:
        
    sess.run(tf.global_variables_initializer())

    print("Initialized session")
        if(model_exists_already):
            print("loading existing model")
            saver.restore(sess, model_path)
        
       
        f_enc = (holdout_data)
        encoder_useful_state = sess.run(encoder_concat_everything, f_enc)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1004560800805544, 'fraction_numerical': 0.012201622934312623, 'mean_word_length': 3.7520405291303125, 'pattern_counts': {'":': 1, '<': 8, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '240419', 'license': 'Apache-2.0', 'n_tokens_mistral': 5814, 'n_tokens_neox': 5553, 'n_words': 1020, 'path': 'absa_adapt_model/word_drop_bridge/.ipynb_checkpoints/Seq2seq-checkpoint.ipynb', 'repo_name': 'shantanuj/Fine_Grained_Sen_Extraction', 'size': '16883'}