<jupyter_start><jupyter_text>Model<jupyter_code># Building Input Placeholders
def model_inputs():
    '''
    Returns : input_,target,learning_rate,keep_prob,source_seq_length,target_seq_length,max_target_seq_length
    '''
    input_ = tf.placeholder(dtype=tf.int32,shape=(None,None),name="inputs")
    target = tf.placeholder(dtype=tf.int32,shape=(None,None),name="target")
    
    learning_rate = tf.placeholder(dtype=tf.float32,name="learning_rate")
    keep_prob = tf.placeholder(dtype=tf.float32,name="keep_prob")
    
    source_seq_length = tf.placeholder(dtype=tf.int32,shape=(None,),name="source_seq_length")
    target_seq_length = tf.placeholder(dtype=tf.int32,shape=(None,),name="target_seq_length")
    
    max_target_seq_length = tf.reduce_max(target_seq_length,name="max_target_seq_length")
    return input_,target,learning_rate,keep_prob,source_seq_length,target_seq_length,max_target_seq_length
#Process decoder input
def process_decoder_input(target_data,vocab_to_int,batch_size):
    
    strided_target = tf.strided_slice(target_data,(0,0),(batch_size,-1),(1,1))
    go = tf.fill(value=vocab_to_int["<GO>"],dims=(batch_size,1))
    decoder_input = tf.concat((go,strided_target),axis=1)
    return decoder_input
def encoding_layer(embeded_rnn_input,rnn_size,keep_prob,num_layers,batch_size,source_sequence_length):

    def get_lstm(rnn_size,keep_prob=0.7):
        lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)
        drop = tf.contrib.rnn.DropoutWrapper(lstm,input_keep_prob=keep_prob)
        return drop
    #     forward lstm layer
    cell_fw = tf.contrib.rnn.MultiRNNCell([get_lstm(rnn_size,keep_prob) for _ in range(num_layers)])

    #     backward lstm layer
    cell_bw = tf.contrib.rnn.MultiRNNCell([get_lstm(rnn_size,keep_prob) for _ in range(num_layers)])
    
    ((encoder_fw_outputs,
              encoder_bw_outputs),
             (encoder_fw_state,
              encoder_bw_state)) = tf.nn.bidirectional_dynamic_rnn(cell_fw=cell_fw,cell_bw=cell_bw,inputs=embeded_rnn_input,
                                    sequence_length=source_sequence_length,dtype=tf.float32)
                                                                     
    encoder_outputs = tf.concat((encoder_fw_outputs, encoder_bw_outputs), 2)
    
    encoder_states = []
    
    for i in range(num_layers):
        if isinstance(encoder_fw_state[i],tf.contrib.rnn.LSTMStateTuple):
            encoder_state_c = tf.concat(values=(encoder_fw_state[i].c,encoder_bw_state[i].c),axis=1,name="encoder_fw_state_c")
            encoder_state_h = tf.concat(values=(encoder_fw_state[i].h,encoder_bw_state[i].h),axis=1,name="encoder_fw_state_h")
            encoder_state = tf.contrib.rnn.LSTMStateTuple(c=encoder_state_c, h=encoder_state_h)
        elif isinstance(encoder_fw_state[i], tf.Tensor):
            encoder_state = tf.concat(values=(encoder_fw_state[i], encoder_bw_state[i]), axis=1, name='bidirectional_concat')
        
        encoder_states.append(encoder_state)
    
    encoder_states = tuple(encoder_states)
    
    return encoder_outputs,encoder_states
def training_decoder(dec_embed_input,decoder_cell,encoder_state, output_layer,
                     target_sequence_length,max_target_length):
    
    
    helper = tf.contrib.seq2seq.TrainingHelper(dec_embed_input,target_sequence_length)
    
    decoder = tf.contrib.seq2seq.BasicDecoder(decoder_cell,helper,initial_state=encoder_state,
                                              output_layer=output_layer)
    
    
    (final_outputs, final_state, final_sequence_lengths) = tf.contrib.seq2seq.dynamic_decode(decoder=decoder,impute_finished=True,
                                                     maximum_iterations=max_target_length)
    
    return final_outputs
def inference_decoder(embeddings,decoder_cell,encoder_state,output_layer,vocab_to_int,
                      max_target_length,batch_size):
    
    start_tokens = tf.tile(tf.constant(dtype=tf.int32,value=[vocab_to_int["<GO>"]]),
                           multiples=[batch_size],name="start_tokens")
    
    helper = tf.contrib.seq2seq.GreedyEmbeddingHelper(embeddings,
                                                      start_tokens=start_tokens,
                                                      end_token=vocab_to_int["<EOS>"])
    
    decoder = tf.contrib.seq2seq.BasicDecoder(decoder_cell,helper,initial_state=encoder_state,
                                              output_layer=output_layer)
    
    (final_outputs, final_state, final_sequence_lengths) = tf.contrib.seq2seq.dynamic_decode(decoder,impute_finished=True,
                                                  maximum_iterations=max_target_length)
    return final_outputs
def decoding_layer(target_inputs,encoder_state,embedding,vocab_to_int,rnn_size,target_sequence_length,max_target_length,
                   batch_size,num_layers):
    
    def get_lstm(rnn_size,keep_prob=0.7):
        rnn_size = 2 * rnn_size
        lstm = tf.contrib.rnn.BasicLSTMCell(rnn_size)
        drop = tf.contrib.rnn.DropoutWrapper(lstm,input_keep_prob=keep_prob)
        return drop
    vocab_len = len(vocab_to_int)
    decoder_cell = tf.contrib.rnn.MultiRNNCell([get_lstm(rnn_size) for _ in range(num_layers)])
    output_layer = Dense(vocab_len,kernel_initializer=tf.truncated_normal_initializer(stddev=0.1))
    
    
    embed = tf.nn.embedding_lookup(embedding,target_inputs)
    
    with tf.variable_scope("decoding"):
        
        training_logits = training_decoder(embed,decoder_cell,encoder_state,output_layer,
                                         target_sequence_length,max_target_length)
    
        
    with tf.variable_scope("decoding",reuse=True):
        
        inference_logits = inference_decoder(embedding,decoder_cell,encoder_state,output_layer,vocab_to_int,
                                          max_target_length,batch_size)
    
    return training_logits, inference_logits
def seq2seq_model(source_input,target_input,embeding_matrix,vocab_to_int,source_sequence_length,
                  target_sequence_length,max_target_length, rnn_size,keep_prob,num_layers,batch_size):
    '''
    Params : source_input,target_input,embeding_matrix,vocab_to_int,source_sequence_length,
                  target_sequence_length,max_target_length, rnn_size,keep_prob,num_layers,batch_size
    
    Return : training_logits, inference_logits
    '''
    embedings = embeding_matrix
    embed = tf.nn.embedding_lookup(embedings,source_input)
    
    encoder_output,encoder_states = encoding_layer(embed,rnn_size,keep_prob,num_layers,
                                                   batch_size,source_sequence_length)
    
    training_logits, inference_logits = decoding_layer(target_input,encoder_states,embedings,
                                                                vocab_to_int,rnn_size,target_sequence_length,
                                                                max_target_length,batch_size,num_layers)
    
    return training_logits, inference_logits<jupyter_output><empty_output><jupyter_text>Batching<jupyter_code># Sorting the text and summary for better padding
# sort based on length of length of text
def sort_text_summary(texts,summaries):
    text_length = [(i,text,len(text)) for i,text in enumerate(texts)]
    text_length.sort(key=operator.itemgetter(2))
    
    sorted_text = [text for i,text,length in text_length]
    sorted_summary = []
    for i,text,length in text_length:
        sorted_summary.append(summaries[i])
    return sorted_text,sorted_summary
sorted_text, sorted_summary = sort_text_summary(encoded_sources,encoded_targets)
len(sorted_text)
# Padding batches
def pad_sentence_batch(sentence_batch):
    max_length = max([len(sent) for sent in sentence_batch])
    padded_sentences = []
    for sent in sentence_batch:
        sent_len = len(sent)
        if len(sent) < max_length:
            padded_sentences.append(sent + [vocab_to_int["<PAD>"] for _ in range(max_length - sent_len)])
        else:
            padded_sentences.append(sent)
    return padded_sentences
def get_batches(encoded_sources, encoded_targets, batch_size):
    
    '''
    Params : encoded_sources, encoded_targets, batch_size
    Return : text_batch,summary_batch,source_seq_len,target_seq_len
    '''
    
    sorted_text, sorted_summary = sort_text_summary(encoded_sources,encoded_targets)
    
    batch_count = len(sorted_text)//batch_size
    
    for i in range(batch_count):
        start = i * batch_size
        end = start + batch_size
        
        text_batch = np.array(pad_sentence_batch(sorted_text[start:end]))
        summary_batch = np.array(pad_sentence_batch(sorted_summary[start:end]))
        
        source_seq_len = [len(sent) for sent in text_batch]
        target_seq_len = [len(sent) for sent in summary_batch]
        
        yield (text_batch,summary_batch,source_seq_len,target_seq_len)
        
# Hyperparametrs
epochs = 3
batch_size = 512
rnn_size = 100
num_layers = 3
learn_rate = 0.01
keep_probability = 0.75

#Model save path
save_path = 'models/model'

display_step = 5
# Build Graph

train_graph = tf.Graph()
with train_graph.as_default():
    
    # Load the model inputs   
    input_,target,learning_rate,keep_prob,source_seq_length,target_seq_length,max_target_seq_length = model_inputs()
    
    # Create the training and inference logits
    training_logits, inference_logits = seq2seq_model(input_,target,embeding_matrix,vocab_to_int,source_seq_length,target_seq_length,
                  max_target_seq_length,rnn_size,keep_probability,num_layers,batch_size)
    
    # Create tensors for the training logits and inference logits
    training_logits = tf.identity(training_logits.rnn_output, name='logits')
    inference_logits = tf.identity(inference_logits.sample_id, name='predictions')
    
    masks = tf.sequence_mask(target_seq_length, max_target_seq_length, dtype=tf.float32, name='masks')

    with tf.name_scope("optimization"):
        cost = tf.contrib.seq2seq.sequence_loss(training_logits,target,masks)
        optimizer=tf.train.AdamOptimizer(learning_rate)
        
        # Gradient Clipping
        gradients = optimizer.compute_gradients(cost)
        capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients if grad is not None]
        train_op = optimizer.apply_gradients(capped_gradients)<jupyter_output><empty_output><jupyter_text>Training<jupyter_code># Accuracy
def get_accuracy(target, logits):
    """
    Calculate accuracy
    """
    max_seq = max(target.shape[1], logits.shape[1])
    if max_seq - target.shape[1]:
        target = np.pad(
            target,
            [(0,0),(0,max_seq - target.shape[1])],
            'constant')
    if max_seq - logits.shape[1]:
        logits = np.pad(
            logits,
            [(0,0),(0,max_seq - logits.shape[1])],
            'constant')

    return np.mean(np.equal(target, logits))
# Split data to training and validation sets (1 Batch for Validation and rest for Training)
train_source = sorted_text[batch_size:]
train_target = sorted_summary[batch_size:]
valid_source = sorted_text[:batch_size]
valid_target = sorted_summary[:batch_size]
(valid_text_batch,valid_summary_batch,valid_source_seq_len,valid_target_seq_len) = next(get_batches(valid_source,valid_target,batch_size))
print(len(valid_source_seq_len),len(valid_target_seq_len))
with tf.Session(graph=train_graph) as sess:
    sess.run(tf.global_variables_initializer())
    
    for epoch_i in range(epochs):
        for batch_i,(text_batch,summary_batch,source_seq_len,target_seq_len) in enumerate(
            get_batches(train_source,train_target,batch_size)):
            
           
            _, loss = sess.run([train_op,cost],
                              feed_dict={
                                  input_ : text_batch,
                                  target : summary_batch,
                                  learning_rate:learn_rate,
                                  keep_prob : keep_probability,
                                  source_seq_length : source_seq_len,
                                  target_seq_length : target_seq_len
                              })
            
            if batch_i % display_step == 0 and batch_i > 0:
                
                batch_train_logits = sess.run(inference_logits,
                                             feed_dict={
                                                 input_: text_batch,
                                                 source_seq_length: source_seq_len,
                                                 target_seq_length: target_seq_len,
                                                 keep_prob: 1.0
                                             })
                
                batch_valid_logits = sess.run(inference_logits,
                                             feed_dict={
                                                 input_: valid_text_batch,
                                                 source_seq_length: valid_source_seq_len,
                                                 target_seq_length: valid_target_seq_len,
                                                 keep_prob: 1.0
                                             })
                
                train_accuracy = get_accuracy(summary_batch,batch_train_logits)
                valid_accuracy = get_accuracy(valid_summary_batch,batch_valid_logits)
                
                print('Epoch {:>3} Batch {:>4}/{} - Train Accuracy: {:>6.4f}, Validation Accuracy: {:>6.4f}, Loss: {:>6.4f}'
                      .format(epoch_i, batch_i, len(sorted_text) // batch_size, train_accuracy, valid_accuracy, loss))
                
    # Save Model
    saver = tf.train.Saver()
    saver.save(sess, save_path)
    print('Model Trained and Saved')<jupyter_output>Epoch   0 Batch    5/1110 - Train Accuracy: 0.7544, Validation Accuracy: 0.7522, Loss: 2.0595
Epoch   0 Batch   10/1110 - Train Accuracy: 0.8000, Validation Accuracy: 0.7522, Loss: 1.4508
Epoch   0 Batch   15/1110 - Train Accuracy: 0.7388, Validation Accuracy: 0.7522, Loss: 1.7768
Epoch   0 Batch   20/1110 - Train Accuracy: 0.5776, Validation Accuracy: 0.7258, Loss: 1.9066
Epoch   0 Batch   25/1110 - Train Accuracy: 0.2160, Validation Accuracy: 0.5138, Loss: 1.4715
Epoch   0 Batch   30/1110 - Train Accuracy: 0.5788, Validation Accuracy: 0.6944, Loss: 0.9707
Epoch   0 Batch   35/1110 - Train Accuracy: 0.7399, Validation Accuracy: 0.7314, Loss: 1.0079
Epoch   0 Batch   40/1110 - Train Accuracy: 0.8546, Validation Accuracy: 0.7539, Loss: 0.9211
Epoch   0 Batch   45/1110 - Train Accuracy: 0.8494, Validation Accuracy: 0.7539, Loss: 0.9985
Epoch   0 Batch   50/1110 - Train Accuracy: 0.8483, Validation Accuracy: 0.7539, Loss: 0.9754
Epoch   0 Batch   55/1110 - Train Accuracy: 0.8666, Validati[...]
{'fraction_non_alphanumeric': 0.0880156113316735, 'fraction_numerical': 0.021600161496534553, 'mean_word_length': 2.6266471449487554, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 55, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '568768', 'license': 'Apache-2.0', 'n_tokens_mistral': 4823, 'n_tokens_neox': 4532, 'n_words': 677, 'path': 'Text Summarization.ipynb', 'repo_name': 'rishiabhishek/Text-Summarization', 'size': '14861'}