<jupyter_start><jupyter_text>Class Definitions Data Model:- Raw data- Vectorizer- Vectorized Data- Data generator<jupyter_code>class RawSurnames(object):
    def __init__(self, data_path, delimiter=","):
        self.data = pd.read_csv(data_path, delimiter=delimiter)

    def get_data(self, filter_to_nationality=None):
        if filter_to_nationality is not None:
            return self.data[self.data.nationality.isin(filter_to_nationality)]
        return self.data

# vectorizer

class SurnamesVectorizer(object):
    def __init__(self, surname_vocab, nationality_vocab, max_seq_length):
        self.surname_vocab = surname_vocab
        self.nationality_vocab = nationality_vocab
        self.max_seq_length = max_seq_length
        
    def save(self, filename):
        vec_dict = {"surname_vocab": self.surname_vocab.get_serializable_contents(),
                    "nationality_vocab": self.nationality_vocab.get_serializable_contents(),
                    'max_seq_length': self.max_seq_length}

        with open(filename, "w") as fp:
            json.dump(vec_dict, fp)
        
    @classmethod
    def load(cls, filename):
        with open(filename, "r") as fp:
            vec_dict = json.load(fp)

        vec_dict["surname_vocab"] = Vocabulary.deserialize_from_contents(vec_dict["surname_vocab"])
        vec_dict["nationality_vocab"] = Vocabulary.deserialize_from_contents(vec_dict["nationality_vocab"])
        return cls(**vec_dict)

    @classmethod
    def fit(cls, surname_df):
        surname_vocab = Vocabulary(use_unks=False,
                                   use_mask=True,
                                   use_start_end=True,
                                   start_token=START_TOKEN,
                                   end_token=END_TOKEN)

        nationality_vocab = Vocabulary(use_unks=False, use_start_end=False, use_mask=False)

        max_seq_length = 0
        for index, row in surname_df.iterrows():
            surname_vocab.add_many(row.surname)
            nationality_vocab.add(row.nationality)

            if len(row.surname) > max_seq_length:
                max_seq_length = len(row.surname)
        max_seq_length = max_seq_length + 2

        return cls(surname_vocab, nationality_vocab, max_seq_length)

    @classmethod
    def fit_transform(cls, surname_df, split='train'):
        vectorizer = cls.fit(surname_df)
        return vectorizer, vectorizer.transform(surname_df, split)
    
    def transform(self, surname_df, split='train'):

        df = surname_df[surname_df.split==split].reset_index()
        n_data = len(df)
        
        x_surnames = np.zeros((n_data, self.max_seq_length), dtype=np.int64)
        y_surnames = np.ones((n_data, self.max_seq_length), dtype=np.int64) * IGNORE_INDEX_VALUE
        x_nationalities = np.zeros(n_data, dtype=np.int64)

        for index, row in df.iterrows():
            vectorized_surname = list(self.surname_vocab.map(row.surname, 
                                                             include_start_end=True))
            x_part = vectorized_surname[:-1]
            y_part = vectorized_surname[1:]
            x_surnames[index, :len(x_part)] = x_part
            y_surnames[index, :len(y_part)] = y_part
            x_nationalities[index] = self.nationality_vocab[row.nationality]

        return VectorizedSurnames(x_surnames, x_nationalities, y_surnames)

# vec data

class VectorizedSurnames(Dataset):
    def __init__(self, x_surnames, x_nationalities, y_surnames):
        self.x_surnames = x_surnames
        self.x_nationalities = x_nationalities
        self.y_surnames = y_surnames

    def __len__(self):
        return len(self.x_surnames)

    def __getitem__(self, index):
        return {'x_surnames': self.x_surnames[index],
                'x_nationalities': self.x_nationalities[index],
                'y_surnames': self.y_surnames[index],
                'x_lengths': len(self.x_surnames[index].nonzero()[0])}

# data generator
def generate_batches(dataset, batch_size, shuffle=True,
                     drop_last=True, use_cuda=False):
        
    dataloader = DataLoader(dataset=dataset, batch_size=batch_size,
                            shuffle=shuffle, drop_last=drop_last)

    for data_dict in dataloader:
        out_data_dict = {}
        for name, tensor in data_dict.items():
            out_data_dict[name] = Variable(tensor)
            if use_cuda:
                out_data_dict[name] = out_data_dict[name].cuda()
        yield out_data_dict<jupyter_output><empty_output><jupyter_text>Class definitions for the model<jupyter_code>def new_parameter(*size):
    out = Parameter(torch.FloatTensor(*size))
    torch.nn.init.xavier_normal(out)
    return out

def column_gather(y_out, x_lengths):
    '''Get a specific vector from each batch datapoint in `y_out`.

    More precisely, iterate over batch row indices, get the vector that's at
    the position indicated by the corresponding value in `x_lengths` at the row
    index. 

    Args:
        y_out (torch.FloatTensor, torch.cuda.FloatTensor)
            shape: (batch, sequence, feature)
        x_lengths (torch.LongTensor, torch.cuda.LongTensor)
            shape: (batch,)

    Returns:
        y_out (torch.FloatTensor, torch.cuda.FloatTensor)
            shape: (batch, feature)
    '''
    x_lengths = x_lengths.long().data.cpu().numpy() - 1
    # alternatively:
    # out = []
    # for batch_index, column_index in enumerate(x_lengths):
    #     out.append(y_out[batch_index, column_index])
    # return torch.stack(out)
    return torch.stack([y_out[batch_index, column_index]
                        for batch_index, column_index in enumerate(x_lengths)])

class ExplicitRNN(nn.Module):
    def __init__(self, input_size, hidden_size, batch_first=False):
        super(ExplicitRNN, self).__init__()
        self.W_in2hid = new_parameter(input_size, hidden_size)
        self.W_hid2hid = new_parameter(hidden_size, hidden_size)
            
        self.b_hid = new_parameter(1, hidden_size)
        
        self.hidden_size = hidden_size

        self.batch_first = batch_first
    
    def _compute_next_hidden(self, x, h):
        return F.tanh(x.matmul(self.W_in2hid) + 
                      h.matmul(self.W_hid2hid) + 
                      self.b_hid)

    def forward(self, x_in, hid_t=None):
        if self.batch_first:
            batch_size, seq_size, feat_size = x_in.size()
            x_in = x_in.permute(1, 0, 2)
        else:
            seq_size, batch_size, feat_size = x_in.size()

        hiddens = []
        if hid_t is None:
            hid_t = Variable(torch.zeros((batch_size, self.hidden_size)))
        
        if x_in.is_cuda:
            hid_t = hid_t.cuda()
            
        for t in range(seq_size):
            x_t = x_in[t]
            hid_t = self._compute_next_hidden(x_t, hid_t)
            
            hiddens.append(hid_t)
        hiddens = torch.stack(hiddens)

        if self.batch_first:
            hiddens = hiddens.permute(1, 0, 2)

        return hiddens
    
class CharNN(nn.Module):
    def __init__(self, embedding_size, in_vocab_size, out_vocab_size, hidden_size, num_conditioning_states,
                 batch_first=False):
        super(CharNN, self).__init__()
        
        self.emb = nn.Embedding(embedding_dim=embedding_size, 
                                num_embeddings=in_vocab_size, 
                                padding_idx=0)
        self.conditional_emb = nn.Embedding(embedding_dim=hidden_size, 
                                            num_embeddings=num_conditioning_states)
        self.fc = nn.Linear(in_features=hidden_size, out_features=out_vocab_size)
        self.rnn = ExplicitRNN(input_size=embedding_size, hidden_size=hidden_size, 
                               batch_first=batch_first)
    
    def forward(self, x_in, state_in, x_lengths=None, apply_softmax=False):
        x_in = self.emb(x_in)
        state_in = self.conditional_emb(state_in)
        y_out = self.rnn(x_in, state_in)

        dim0, dim1, dim2 = y_out.size()
        y_out = y_out.contiguous().view(-1, dim2)

        y_out = self.fc(y_out)

        # optionally apply the softmax
        if apply_softmax:
            y_out = F.softmax(y_out, dim=1)

        y_out = y_out.view(dim0, dim1, -1)
        
        return y_out
    
def normalize_sizes(net_output, y_true):
    net_output = net_output.cpu()
    y_true = y_true.cpu()
    if len(net_output.size()) == 3:
        net_output.contiguous()
        net_output = net_output.view(-1, net_output.size(2))
    if len(y_true.size()) == 2:
        y_true.contiguous()
        y_true = y_true.view(-1)
    return net_output, y_true

def compute_accuracy(y_pred, y_true, mask_index):
    y_pred, y_true = normalize_sizes(y_pred, y_true)

    _, y_pred_indices = y_pred.cpu().max(dim=1)
    
    correct_indices = torch.eq(y_pred_indices, y_true).float()
    valid_indices = torch.ne(y_true, mask_index).float()
    
    n_correct = (correct_indices * valid_indices).sum().data[0]
    n_valid = valid_indices.sum().data[0]

    return n_correct / n_valid * 100
args = Namespace(
    surname_csv="data/surnames.csv",
    batch_size = 128,
    cuda=False,
    learning_rate=0.001,
    num_epochs=100,
    load_zoo_model=True,
    zoo={
        'filename': 'modelzoo/charnn_emb16_hid64_surnames_conditionally_predict.state',
        'vocab': 'modelzoo/surnames_classify.vocab',
        'comments': 'pre-trained surname conditioned sequence prediction (& conditioned generation)',
        'parameters': {
            'embedding_size': 16,
            'hidden_size': 64
        }
    }
)


args.cuda = args.cuda and torch.cuda.is_available()

if args.cuda:
    print("CUDA is enabled")
else:
    print("CUDA is not enabled")
# optional: set this to false to learn from scratch!
# args.load_zoo_model = False
raw_data = RawSurnames(args.surname_csv).get_data()

if os.path.exists(args.zoo['vocab']):
    vectorizer = SurnamesVectorizer.load(args.zoo['vocab'])
    print("Loading vectorizer!")
else:
    vectorizer = SurnamesVectorizer.fit(raw_data)
    print("Creating a new vectorizer.")
    
    
train_dataset = vectorizer.transform(raw_data, split='train')
test_dataset = vectorizer.transform(raw_data, split='test')


zoo_params = args.zoo['parameters']

net = CharNN(embedding_size=zoo_params['embedding_size'], 
             hidden_size=zoo_params['hidden_size'],
             in_vocab_size=len(vectorizer.surname_vocab), 
             out_vocab_size=len(vectorizer.surname_vocab), 
             num_conditioning_states=len(vectorizer.nationality_vocab),
             batch_first=True)

if args.load_zoo_model and os.path.exists(args.zoo['filename']):
    print("Loading state dict!")
    net.load_state_dict(torch.load(args.zoo['filename'], 
                                   map_location=lambda storage, loc: storage))
else:
    print("Using newly initiated network!")
if args.cuda:
    print("CUDA mode enabled")
    net = net.cuda()
else:
    print("CUDA mode not enabled")
    net = net.cpu()
    
# optimizer 

optimizer = torch.optim.Adam(net.parameters(), lr=args.learning_rate)

# loss function

def sequence_loss(y_pred, y_true, mask_index):
    y_pred, y_true = normalize_sizes(y_pred, y_true)
    return F.cross_entropy(y_pred, y_true, ignore_index=mask_index)

# progress bars

epoch_bar = tqdm_notebook(desc='epochs', total=args.num_epochs, position=1)

num_train_batches = len(train_dataset) // args.batch_size
train_bar = tqdm_notebook(desc='training', total=num_train_batches, position=2)

num_test_batches = len(test_dataset) // args.batch_size
test_bar = tqdm_notebook(desc='test', total=num_test_batches, position=3)

# history

train_loss_history = []
train_accuracy_history = []

test_loss_history = []
test_accuracy_history = []


try:
    for _ in range(args.num_epochs):
        batch_generator = generate_batches(train_dataset, batch_size=args.batch_size)
        
        per_epoch_loss = []
        per_epoch_accuracy = []
        
        net.train()
            
        for batch_dict in batch_generator:
            # step 1
            optimizer.zero_grad()
            
            # step 2
            y_pred = net(batch_dict['x_surnames'], 
                         batch_dict['x_nationalities'],
                         batch_dict['x_lengths'])
            y_target = batch_dict['y_surnames'] 
            
            # step 3
            loss = sequence_loss(y_pred, y_target, IGNORE_INDEX_VALUE)

            # step 4
            loss.backward()
            optimizer.step()

            # bonus steps: bookkeeping
            per_epoch_loss.append(loss.cpu().data[0])        
        
            accuracy = compute_accuracy(y_pred, batch_dict['y_surnames'], IGNORE_INDEX_VALUE)
            per_epoch_accuracy.append(accuracy)

            train_bar.update()
            
            train_bar.set_postfix(loss=per_epoch_loss[-1], 
                                  accuracy=per_epoch_accuracy[-1])
            
        train_loss_history.append(np.mean(per_epoch_loss))
        train_accuracy_history.append(np.mean(per_epoch_accuracy))
        
        # loop over test dataset
        
        batch_generator = generate_batches(test_dataset, batch_size=args.batch_size)
        
        per_epoch_loss = []
        per_epoch_accuracy = []
            
        # set it to eval mode; this turns stochastic functions off
        net.eval()
            
        for batch_dict in batch_generator:
            # step 1: compute output
            y_pred = net(batch_dict['x_surnames'], 
                         batch_dict['x_nationalities'],
                         batch_dict['x_lengths'])
            y_target = batch_dict['y_surnames'] 
            
            # step 2: compute metrics
            loss = sequence_loss(y_pred, y_target, IGNORE_INDEX_VALUE)
            per_epoch_loss.append(loss.cpu().data[0])
          
            accuracy = compute_accuracy(y_pred, batch_dict['y_surnames'], IGNORE_INDEX_VALUE)
            per_epoch_accuracy.append(accuracy)

            test_bar.update()
            
            test_bar.set_postfix(loss=per_epoch_loss[-1], 
                                 accuracy=per_epoch_accuracy[-1])
            
        test_loss_history.append(np.mean(per_epoch_loss))
        test_accuracy_history.append(np.mean(per_epoch_accuracy))
        
        # update bars
        
        epoch_bar.set_postfix(train_loss=train_loss_history[-1], 
                              train_accuracy=train_accuracy_history[-1],
                              test_loss=test_loss_history[-1],
                              test_accuracy=test_accuracy_history[-1])
        epoch_bar.update()
        test_bar.n = 0
        train_bar.n = 0
        
except KeyboardInterrupt:
    print("...")<jupyter_output>CUDA mode not enabled<jupyter_text>SamplingIn the plain prediction notebook, the exercise was to create the sampling routine. Below is code already written to sample, so you can see how to handle the conditional nature.  As an exercise, you should consider what it would mean to interpolate between two conditioning vectors!  For instance, you could take the Irish and Chinese embeddings and average them (which is the same thing as multiplying each vector by 0.5 and adding them together!). Or you could weight them 0.3 and 0.7 and then add them together.  This is referred to as interpolation.  The weights should add up to 1 to be a valid interpolation.<jupyter_code>def sample(emb, rnn, fc, h_t=None, idx_t=None, n=20, temp=1):
    hiddens = [h_t]
    indices = [idx_t]
    out_dists = []
    
    for t in range(n):
        x_t = emb(idx_t)
        h_t = rnn._compute_next_hidden(x_t, h_t)
        
        y_t = fc(h_t)
        y_t = F.softmax( y_t / temp, dim=1)
        idx_t = torch.multinomial(y_t, 1)[:, 0]
        
        
        hiddens.append(h_t)
        indices.append(idx_t)
        out_dists.append(y_t)
     
    indices = torch.stack(indices).squeeze().permute(1, 0)
    return indices

def long_variable(indices):
    out = Variable(torch.LongTensor(indices))
    return out

def make_initial_hidden(batch_size, hidden_size):
    out = Variable(torch.ones(batch_size, hidden_size))
    return out

def make_initial_x(batch_size, vectorizer):
    out = Variable(torch.ones(batch_size) * vectorizer.surname_vocab.start_index).long()
    return out

def decode_one(vectorizer, seq):
    out = []
    for i in seq:
        if vectorizer.surname_vocab.start_index == i:
            continue
        if vectorizer.surname_vocab.end_index == i:
            return ''.join(out)
        out.append(vectorizer.surname_vocab.lookup(i))
    return ''.join(out)
            
def decode_matrix(vectorizer, mat):
    mat = mat.cpu().data.numpy()
    return [decode_one(vectorizer, mat[i]) for i in range(len(mat))]

def n_random_nationalities(n):
    keys = np.random.choice(vectorizer.nationality_vocab.keys(), size=n, replace=True)
    indices = long_variable([vectorizer.nationality_vocab[key] for key in keys])
    return keys, indices

def sample_n(n=10, temp=0.8):
    init_names, init_vector = n_random_nationalities(n)
    init_vector = net.conditional_emb(init_vector)
    samples = decode_matrix(vectorizer, 
                            sample(net.emb, net.rnn, net.fc, 
                                   init_vector, 
                                   make_initial_x(n, vectorizer),
                                   temp=temp))
    return list(zip(init_names, samples))

def sample_n_for_nationality(nationality, n=10, temp=0.8):
    assert nationality in vectorizer.nationality_vocab.keys(), 'not a nationality we trained on'
    keys = [nationality] * n
    init_vector = long_variable([vectorizer.nationality_vocab[key] for key in keys])
    init_vector = net.conditional_emb(init_vector)
    samples = decode_matrix(vectorizer, 
                        sample(net.emb, net.rnn, net.fc, 
                               init_vector, 
                               make_initial_x(n, vectorizer),
                               temp=temp))
    return list(zip(keys, samples))
sample_n_for_nationality('japanese', n=10)
sample_n()
sample_n(30, 0.8)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09720621329381415, 'fraction_numerical': 0.006147428508699709, 'mean_word_length': 2.442932728647014, 'pattern_counts': {'":': 2, '<': 12, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '582075', 'license': 'MIT', 'n_tokens_mistral': 6062, 'n_tokens_neox': 5861, 'n_words': 1063, 'path': 'day_2/03-Char-RNN-Conditionally-Predict-Surnames.ipynb', 'repo_name': 'joosthub/pytorch-nlp-tutorial-ny2018', 'size': '18219'}