<jupyter_start><jupyter_text>[View in Colaboratory](https://colab.research.google.com/github/jsedoc/Sharvin-Shah-Spring-2018/blob/master/tree2seq_colab.ipynb)<jupyter_code>import sys
print(sys.version_info)

!pip install http://download.pytorch.org/whl/cu90/torch-0.3.0.post4-cp36-cp36m-linux_x86_64.whl
!pip install tqdm nltk
import torch

print(torch.cuda.device_count())
use_cuda = torch.cuda.is_available()
use_cuda = 0
!rm -r data synth_t2t_data.zip
!wget http://www.seas.upenn.edu/~joao/synth_t2t_data.zip
!unzip synth_t2t_data.zip
!pwd
from tqdm import tqdm
from copy import deepcopy

PAD = 0
UNK = 1
BOS = 2
EOS = 3

PAD_WORD = '<blank>'
UNK_WORD = '<unk>'
BOS_WORD = '<s>'
EOS_WORD = '</s>'

MAX_LENGTH = 512


import torch
import torch.utils.data as data

from nltk.tree import Tree

datapath = '/content/data/train.orig'

class Dataset(data.Dataset):

    def __len__(self):
        return self.size

    def read_trees(self, filename):
        with open(filename, 'r') as f:
            trees = [Tree.fromstring(line) for line in tqdm(f.readlines())]
        return trees
    
    def read_seqs(self, filename):
        with open(filename, 'r') as f:
            seqs = [line for line in tqdm(f.readlines())]
        return seqs
dataset = Dataset()
trees = dataset.read_trees('/content/data/train.orig')
# print(trees[1])
# print(len(trees))
# creating a dictionary of unique tokens with the assigned id with file input
def create_vocab(filename):
    vocab = dict()
    with open(filename, 'r') as f:
        for token in f.read().split():
            if token not in vocab:
                vocab[token] = 1
            else:
                vocab[token] += 1
    # print(len(vocab.keys()))    
    index = 0
    token_dict = {}            
    for token in vocab.keys():
        token_dict[token] = index
        index += 1
    
    return token_dict

# fetch id of a token from the dictionary  
def fetch_unique_index(token_dict, token):
    return token_dict[token]

# use dictionary of tokens to form a corresponding one hot representation  
def create_one_hot(token_dict):
    one_hot_dict = {}
    vector_dim = len(token_dict.keys())
    for token in token_dict:    
        tensor = torch.zeros(1, vector_dim)
        tensor = tensor.cuda() if use_cuda else tensor
        tensor[0][token_dict[token]] = 1
        one_hot_dict[token] = tensor
    
    return one_hot_dict

# fetch one hot representation for the token
def fetch_one_hot(one_hot_dict, token):
    return one_hot_dict[token]
vocab = create_vocab(datapath)
# print(vocab)
oh_vocab = create_one_hot(vocab)
# print(fetch_one_hot(oh_vocab, '4'))
dataset = Dataset()
trees = dataset.read_trees(datapath)
seqs = dataset.read_seqs(datapath)
print("\n")
print(trees[0])
print(len(trees[0]))
print(seqs[0]) # maybe the seqs has to be split token wise while training
print(len(seqs[0].split()))
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable as Var

PAD = 0
UNK = 1
BOS = 2
EOS = 3

PAD_WORD = '<blank>'
UNK_WORD = '<unk>'
BOS_WORD = '<s>'
EOS_WORD = '</s>'

# module for childsumtreelstm
class ChildSumTreeLSTM(nn.Module):
    def __init__(self, in_dim, mem_dim):
        super(ChildSumTreeLSTM, self).__init__()
        self.in_dim = in_dim
        self.mem_dim = mem_dim
        self.ioux = nn.Linear(self.in_dim, 3 * self.mem_dim)
        self.iouh = nn.Linear(self.mem_dim, 3 * self.mem_dim)
        self.fx = nn.Linear(self.in_dim, self.mem_dim)
        self.fh = nn.Linear(self.mem_dim, self.mem_dim)

    def node_forward(self, inputs, child_c, child_h):
        child_h_sum = torch.sum(child_h, dim=0, keepdim=True)
        # print("dimension of inputs: ", inputs.size())
        # print("dimension of child_h_sum: ", child_h_sum.size())
        iou = self.ioux(inputs) + self.iouh(child_h_sum)
        i, o, u = torch.split(iou, iou.size(1) // 3, dim=1)
        i, o, u = F.sigmoid(i), F.sigmoid(o), F.tanh(u)

        f = F.sigmoid(
            self.fh(child_h) +
            self.fx(inputs).repeat(len(child_h), 1)
        )
        fc = torch.mul(f, child_c)

        c = torch.mul(i, u) + torch.sum(fc, dim=0, keepdim=True)
        h = torch.mul(o, F.tanh(c))
        return c, h

    def forward(self, tree):
      # print(type(tree))
      # print(tree)
      if type(tree) == Tree:
        child_states = []
        for child in tree:
            child_states.append(self.forward(child))
        child_c, child_h = zip(* map(lambda x: x, child_states))
        child_c, child_h = torch.cat(child_c, dim=0), torch.cat(child_h, dim=0)

        if type(tree) == str:
          node_label_repr = Var(fetch_one_hot(oh_vocab, tree))
        else:
          node_label_repr = Var(fetch_one_hot(oh_vocab, tree.label()))
        state = self.node_forward(node_label_repr, child_c, child_h)
      else:
        # print(type(tree))
        if type(tree) == str:
          node_label_repr = Var(fetch_one_hot(oh_vocab, tree))
        child_c = Var(node_label_repr.data.new(1, self.mem_dim).fill_(0.))
        child_h = Var(node_label_repr.data.new(1, self.mem_dim).fill_(0.))
        state = self.node_forward(node_label_repr, child_c, child_h)
      # Using tree.label() instead of torch.zeros() ensures that the device placement is inherited  

      return state



# module for distance-angle similarity
class Similarity(nn.Module):
  def __init__(self, mem_dim, hidden_dim, num_classes):
    super(Similarity, self).__init__()
    self.mem_dim = mem_dim
    self.hidden_dim = hidden_dim
    self.num_classes = num_classes
    self.wh = nn.Linear(2 * self.mem_dim, self.hidden_dim)
    self.wp = nn.Linear(self.hidden_dim, self.num_classes)

  def forward(self, lvec, rvec):
    mult_dist = torch.mul(lvec, rvec)
    abs_dist = torch.abs(torch.add(lvec, -rvec))
    vec_dist = torch.cat((mult_dist, abs_dist), 1)

    out = F.sigmoid(self.wh(vec_dist))
    out = F.log_softmax(self.wp(out), dim=1)
    return out


# putting the whole model together
class SimilarityTreeLSTM(nn.Module):
    def __init__(self, vocab_size, in_dim, mem_dim, hidden_dim, num_classes, sparsity, freeze):
        super(SimilarityTreeLSTM, self).__init__()
        self.emb = nn.Embedding(vocab_size, in_dim, padding_idx=PAD, sparse=sparsity)
        if freeze:
            self.emb.weight.requires_grad = False
        self.childsumtreelstm = ChildSumTreeLSTM(in_dim, mem_dim)
        self.similarity = Similarity(mem_dim, hidden_dim, num_classes)

    def forward(self, ltree, linputs, rtree, rinputs):
        linputs = self.emb(linputs)
        rinputs = self.emb(rinputs)
        lstate, lhidden = self.childsumtreelstm(ltree, linputs)
        rstate, rhidden = self.childsumtreelstm(rtree, rinputs)
        output = self.similarity(lstate, rstate)
        return output
# encoder input size (in_dim) could be anything
# encoder hidden size (mem_dim) would be equal to the dimension of the vector
# encoder hidden size could be anything, decoder hidden size is dependent on final encoder hidden state size
# decoder output size could be anything
cs_treelstm = ChildSumTreeLSTM(71, 128)
print(cs_treelstm)
out = cs_treelstm.forward(trees[1])
out
out[1]
from copy import deepcopy
class DecoderRNN(nn.Module):
    def __init__(self, hidden_size, output_size):
        super(DecoderRNN, self).__init__()
        self.hidden_size = hidden_size

        self.embedding = nn.Embedding(output_size, hidden_size)
        self.gru = nn.GRU(hidden_size, hidden_size)
        self.out = nn.Linear(hidden_size, output_size)
        self.softmax = nn.LogSoftmax()

    def forward(self, input, hidden):
        # print("input size: ", input.size())
        # print("input: ", input)
        # print("hidden size: ", hidden.size())
        output = self.embedding(input).view(1, 1, -1)
        # print("After Embedding, size: ", output.size())
        output = F.relu(output)
        # print("output size after ReLU: ", output.size())
        output, hidden = self.gru(output, hidden)
        output = self.softmax(self.out(output[0]))
        return output, hidden

    def initHidden(self):
        result = Variable(torch.zeros(1, 1, self.hidden_size))
        if use_cuda:
            return result.cuda()
        else:
            return result
def token_to_index(token_dict, filename):
    file_embedding = []
    
    with open(filename, 'r') as f:
        for line in f.readlines()[:2]:
            line_embedding = []
            print("line: ", line)
            tokens = line.split()
            print("tokens: ", tokens)
            for token in tokens:
                line_embedding.append(token_dict[token])
            
            # print("line embedding: ", line_embedding)
            line_tensor = Var(torch.LongTensor(line_embedding).view(-1, 1))
            # print("line tensor: ", line_tensor)
            file_embedding.append(line_tensor)
            # print("file embedding so far: ", file_embedding)
    
    return file_embedding
  
def index_to_token(token_dict, index):
  flipped = {v:k for k,v in token_dict.items()}
  return flipped[index]
token_embedding = token_to_index(vocab, datapath)
token_embedding[1]
out[1].size()
len(vocab.keys())

decoder = DecoderRNN(out[1].size()[1], len(vocab.keys()))
#hidden = decoder.initHidden()
encoder_outputs = out[1]
encoder_outputs = encoder_outputs.unsqueeze(0)
print(encoder_outputs)
for i in range(out[1].size()[1]):
  out, hidden = decoder.forward(token_embedding[1][i], encoder_outputs)
out
import time
import math


def asMinutes(s):
    m = math.floor(s / 60)
    s -= m * 60
    return '%dm %ds' % (m, s)


def timeSince(since, percent):
    now = time.time()
    s = now - since
    es = s / (percent)
    rs = es - s
    return '%s (- %s)' % (asMinutes(s), asMinutes(rs))
from torch import optim

tree_variable = trees
target_variable = token_embedding

input_size = len(vocab.keys()) # here also equal to length of output vocab
hidden_size = 128 # any size can be given here
output_size = input_size

encoder = ChildSumTreeLSTM(input_size, hidden_size)
decoder = DecoderRNN(hidden_size, output_size)

if use_cuda:
  encoder = encoder.cuda()
  decoder = decoder.cuda()

learning_rate = 0.0005
encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(), lr=learning_rate)

criterion = nn.NLLLoss()

MAX_LENGTH = max([len(i) for i in seqs])

# only one example at a time, needs to be called in an iterative fashion
def train(tree_variable, target_variable, 
          encoder, decoder, encoder_optimizer, decoder_optimizer, 
          criterion, max_length=MAX_LENGTH):
  encoder_optimizer.zero_grad()
  decoder_optimizer.zero_grad()
  # print("tree variable: ", tree_variable)
  # print("target variable: ", target_variable)
  input_length = len(tree_variable)
  # print("input_length: ", input_length)
  target_length = len(target_variable)
  # print("target_length: ", target_length)
  encoder_outputs = Var(torch.zeros(max_length, encoder.mem_dim))
  encoder_outputs = encoder_outputs.cuda() if use_cuda else encoder_outputs
  
  loss = 0

  predicted = []
  
  # encode tree
  # print("Encoding begins......")
  #for ei in range(input_length):
    # print("input variable ei size: ", len(input_variable[ei]))
#   encoder_states = encoder(input_variable[ei])
#   encoder_outputs[ei] = encoder_states[0]
  encoder_states = encoder(input_variable)
  encoder_outputs = encoder_states[0]
    
  # decoder serial and accumulate loss

  decoder_input = Var(torch.LongTensor([[0]]))
  decoder_input = decoder_input.cuda() if use_cuda else decoder_input  
  # print("decoder input: ", decoder_input)
  decoder_hidden = encoder_states[1]
  # print("before unsqueeze: ", decoder_hidden.size())
  decoder_hidden = decoder_hidden.unsqueeze(0)
  # print("after unsqueeze: ", decoder_hidden.size())
  use_teacher_forcing = True 

  if use_teacher_forcing:
    # print("Decoding begins...")
    # Teacher forcing: Feed the target as the next input
    for di in range(target_length):
      decoder_output, decoder_hidden = decoder(
                decoder_input, decoder_hidden)
      target_var = Var(torch.LongTensor([[fetch_unique_index(vocab, target_variable[di])]]))
      target_var = target_var.cuda() if use_cuda else target_var
      # print("variable: ", target_var)
      # print("current element: ", target_variable[di])
      # current_target = Var(fetch_one_hot(oh_vocab, target_variable[di]))
      # target_variable[di] = token_embedding[di]
      # print("one hot representation: ", current_target)
      #import pdb;pdb.set_trace()
      loss += criterion(decoder_output, target_var[0][0])
      topv, topi = decoder_output.data.topk(1)
      ni = topi[0][0]
      predicted.append(ni)
      decoder_input = target_var  # Teacher forcing

  # call backward

  loss.backward()

  # update parameters
  
  encoder_optimizer.step()
  decoder_optimizer.step()

  return (loss.data[0] / target_length, predicted)

import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np


def showPlot(points):
  plt.figure()
  fig, ax = plt.subplots()
  # this locator puts ticks at regular intervals
  loc = ticker.MultipleLocator(base=0.2)
  ax.yaxis.set_major_locator(loc)
  plt.plot(points)

print_every = 100
plot_every = 100
start = time.time()
plot_losses = []
print_loss_total = 0  # Reset every print_every
plot_loss_total = 0  # Reset every plot_every
n_iters = 50

iteration_count = 0

for k in range(n_iters):
  for i in range(len(trees)):
    iteration_count += 1
    input_variable = trees[i]

    target_variable = seqs[i].split()

    loss, predicted = train(input_variable, target_variable, encoder,
                            decoder, encoder_optimizer, decoder_optimizer, criterion)
    print_loss_total += loss
    plot_loss_total += loss

    if (i+1) % print_every == 0:
      print('input: ', trees[i])
      print('target: ', seqs[i][:-1])
      print('pred: ', " ".join([index_to_token(vocab, x) for x in predicted]))
      print_loss_avg = print_loss_total / print_every
      print_loss_total = 0
      print('iterations: %d, time elapsed: %s, loss:%.4f' % (iteration_count, timeSince(start, i / (n_iters)),
                                     print_loss_avg))

    if (i+1) % plot_every == 0:
      plot_loss_avg = plot_loss_total / plot_every
      plot_losses.append(plot_loss_avg)
      print(len(plot_losses))
      plot_loss_total = 0  

#showPlot(plot_losses)      
plot_losses
showPlot(plot_losses)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11846545905365383, 'fraction_numerical': 0.009298161030373992, 'mean_word_length': 3.2806603773584904, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '520363', 'license': 'Apache-2.0', 'n_tokens_mistral': 5147, 'n_tokens_neox': 4933, 'n_words': 1126, 'path': 'tree2seq_colab.ipynb', 'repo_name': 'jsedoc/Sharvin-Shah-Spring-2018', 'size': '14519'}