<jupyter_start><jupyter_text>Classify names with character n-grams<jupyter_code>from argparse import Namespace

import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns
from tqdm import tqdm_notebook

%matplotlib inline

plt.style.use('fivethirtyeight')
plt.rcParams['figure.figsize'] = (14, 6)<jupyter_output><empty_output><jupyter_text>Overview of Data/Task- Data compiled by [<NAME>](https://github.com/spro)- Predict nationality from names.- Data consist of 20074 names, 18 categories. - Russian names are dominant (skewing labels)  - We downsample russian names to minimize the skew. Checkout the RNN tutorial for a different approach to handle label bias.```    2750 names_test.csv   10994 names_train.csv```  Args for this example<jupyter_code>args = Namespace(
    surname_csv="data/surnames.csv",
    model_filename="names_mlp_model.pth",
    cuda=False,
    num_epochs=100
)

args.cuda = args.cuda and torch.cuda.is_available()

if args.cuda:
    print("CUDA is enabled")
else:
    print("CUDA is not enabled")<jupyter_output>CUDA is not enabled<jupyter_text>Load names<jupyter_code>name_data = pd.read_csv(args.surname_csv)<jupyter_output><empty_output><jupyter_text>Class Breakdown<jupyter_code>sns.factorplot(data=name_data, y='nationality', 
               kind='count', size=5, aspect=3);
plt.title("Counts per Nationality in the Surnames Dataset");
def fix_imbalance(df):
    median = df.groupby('nationality').agg('count').split.median()
    for nationality in df.nationality.unique():
        
        df_main = df[df.nationality != nationality]
        df_sub = df[df.nationality == nationality].reset_index(drop=True)
        if len(df_sub) > median:
            size = min(len(df_sub), int(median+0.05*len(df_sub)))
            indices = np.random.choice(np.arange(len(df_sub)), 
                                       size=size, 
                                       replace=False)
            df_sub = df_sub.iloc[indices]
        df = pd.concat([df_main, df_sub]).reset_index(drop=True)
    return df
name_data = fix_imbalance(name_data)
sns.factorplot(data=name_data, y='nationality', 
               kind='count', size=5, aspect=3);
plt.title("Counts per Nationality in the Surnames Dataset POST One-off Subsampling");<jupyter_output><empty_output><jupyter_text>Feature Engineering 1. Lowercase<jupyter_code>name_data['features'] = name_data['surname'].apply(lambda x: x.lower())
name_data.head()<jupyter_output><empty_output><jupyter_text>2. Add begin and end tokens<jupyter_code>def add_begin_end_tokens(name):
    # return "^{}$".format(name)
    begin_token_marker = "^"
    end_token_marker = '$'
    return "".join((begin_token_marker, name, end_token_marker))
name_data['features'] = name_data['features'].apply(add_begin_end_tokens)
name_data.head()<jupyter_output><empty_output><jupyter_text>3. Extract Character N-grams<jupyter_code>def extract_n_grams(tokens, n):
    '''
    param tokens: tokenized text
    return: list of n grams
    '''
    return [tokens[i:i + n] for i in range(len(tokens) - n + 1)]
# use trigrams
name_data['features'] = name_data['features'].apply(lambda x: extract_n_grams(x, 3))
name_data.head()<jupyter_output><empty_output><jupyter_text>4. Build vocabulary<jupyter_code>vocabulary  = set()
for name in name_data['features'].values:
    [vocabulary.add(ngram) for ngram in name]        
maxlen = max(name_data['features'].apply(len))
maxlen<jupyter_output><empty_output><jupyter_text>5. Map n-grams to indices<jupyter_code>ngram_to_index = {ngram:i for i, ngram in enumerate(vocabulary, 1)}
def n_grams_to_indices(n_grams):
    unknown_index = len(ngram_to_index) + 1
    return [ngram_to_index.get(ngram, unknown_index) for ngram in n_grams]  
n_grams_to_indices(['^ha', 'had', 'ada', 'dad', 'ad$'])<jupyter_output><empty_output><jupyter_text>6. Pad indices<jupyter_code>def pad_indices(indices, maxlen=20):
    padded = np.zeros(maxlen, dtype=np.int64)
    for i, index in enumerate(indices):
        padded[i] = index
    return padded 
test_name = n_grams_to_indices(['^ha', 'had', 'ada', 'dad', 'ad$'])
pad_indices(test_name, 20)<jupyter_output><empty_output><jupyter_text>7. Encode class labels as numbers<jupyter_code>label_to_number = {y: i for i, y in enumerate(set(name_data['nationality']))}
number_to_label = {i: y for i, y in enumerate(set(name_data['nationality']))}
label_to_number<jupyter_output><empty_output><jupyter_text>8. Consolidate data preperation pipeline<jupyter_code>import torch
def extract_name_features(name, n_grams=3):
    """
    Applies feature extraction pipeline to name
    """
    features = name.lower()
    features = add_begin_end_tokens(features)
    features = extract_n_grams(features, n_grams)
    features = n_grams_to_indices(features)
    features = pad_indices(features)
    features = torch.from_numpy(features).long()
    return features
extract_name_features('Brian')<jupyter_output><empty_output><jupyter_text>Create PyTorch Dataset<jupyter_code>import torch
from torch.utils.data import Dataset, DataLoader
from torch.autograd import Variable
class VectorizedSurnames(Dataset):
    """Name Classification dataset"""

    def __init__(self, data):
        '''
        Args:
            data (pandas.DataFrame)
        '''
        self.data = data
        self.x_surnames = self.data['surname'].apply(lambda x: extract_name_features(x, 3))
        self.y_nationalities = self.data['nationality'].apply(lambda x: label_to_number[x])

    @classmethod
    def from_file(cls, filename, split='train', apply_fix=True):
        df = pd.read_csv(filename).dropna()
        df = df[df.split == split]
        if apply_fix:
            df = fix_imbalance(df)
        return cls(df)
    
    def __len__(self):
        return len(self.x_surnames)

    def __getitem__(self, index):
        return {'x_surnames': self.x_surnames.iloc[index],
                'y_nationalities': self.y_nationalities.iloc[index]}
name_dataset = VectorizedSurnames.from_file(args.surname_csv, split='train', apply_fix=True)
type(name_dataset.x_surnames[0])
dataloader = DataLoader(name_dataset, batch_size=32, shuffle=True)<jupyter_output><empty_output><jupyter_text>Define MLP<jupyter_code>import torch.nn as nn
import torch.nn.functional as F
class MLP(nn.Module):

    def __init__(self, vocab_size, embedding_dim, hidden_dim, output_size, maxlen):
        super(MLP, self).__init__()
        self.embeddings = nn.Embedding(vocab_size, embedding_dim, padding_idx=0)
        self.fc1 = nn.Linear(embedding_dim * maxlen, hidden_dim)
        self.dropout = nn.Dropout(p=0.5)
        self.fc2 = nn.Linear(hidden_dim, output_size)

    def forward(self, x, apply_softmax=False):
        x = self.embeddings(x).view(-1, (embedding_dim * maxlen))
        x = F.relu(self.fc1(x))
        x = self.dropout(x)
        x = self.fc2(x)
        if apply_softmax:
            x = F.softmax(x, dim=1)
        return x
vocab_size = len(vocabulary) + 2 # one for unknown index
embedding_dim = 100 # hidden dim.
hidden_dim = 200
output_size = len(label_to_number) # number of classes
maxlen = 20

classifier = MLP(vocab_size, embedding_dim, hidden_dim, output_size, maxlen)<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>if args.cuda:
    classifier = classifier.cuda()

optimizer = torch.optim.Adam(classifier.parameters())
criterion = nn.CrossEntropyLoss()

loss_history = []
accuracy_history = []

epoch_bar = tqdm_notebook(desc='epochs', total=args.num_epochs)

try:
    for _ in range(args.num_epochs):
        per_epoch_losses = []
        per_epoch_accuracy = []
        for batch in dataloader:
            # Step 1: clear the gradients 
            optimizer.zero_grad()

            # Step 2: apply the model to the data
            x_surnames = Variable(batch['x_surnames'])
            y_nationalities = Variable(batch['y_nationalities'])
            if args.cuda:
                x_surnames = x_surnames.cuda()
                y_nationalities = y_nationalities.cuda()

            y_prediction = classifier(x_surnames)

            # Step 3: compute the loss and propagate the gradients
            
            loss = criterion(y_prediction, y_nationalities)
            loss.backward()

            # Step 4; have optimizer update the model weights
            optimizer.step()
            
            # bonus steps:  bookkeeping
            
            per_epoch_losses.append(loss.data[0])
            
            accuracy = np.mean(y_prediction.cpu().data.numpy().argmax(axis=-1)==y_nationalities.cpu().data.numpy())
            per_epoch_accuracy.append(accuracy)


        loss_history.append(np.mean(per_epoch_losses))
        accuracy_history.append(np.mean(per_epoch_accuracy))
        epoch_bar.set_postfix(loss=loss_history[-1], accuracy=accuracy_history[-1])
        epoch_bar.update()
except KeyboardInterrupt:
    print("...")

_, ax = plt.subplots(1, 1)
ax.plot(loss_history, color='#942174', label='loss')
plt.legend(bbox_to_anchor=(0.86, 0.7))
ax.grid(False)
ax.set_xlabel("Epoch Index")
ax.set_ylabel("Loss Value")
ax2 = plt.twinx()
ax2.set_ylabel("Accuracy Value")
ax2.plot(accuracy_history, color='#e5a930', label='accuracy')
plt.legend(bbox_to_anchor=(0.9, 0.8))<jupyter_output><empty_output><jupyter_text>save model<jupyter_code>torch.save(classifier.state_dict(), args.model_filename)<jupyter_output><empty_output><jupyter_text>load model<jupyter_code>classifier.load_state_dict(torch.load(args.model_filename))<jupyter_output><empty_output><jupyter_text>Evaluate on test set<jupyter_code>from sklearn.metrics import confusion_matrix, accuracy_score


name_dataset_test = VectorizedSurnames.from_file(args.surname_csv, split='test')
dataloader_test = DataLoader(name_dataset_test, batch_size=16, shuffle=True, num_workers=0)

if args.cuda:
    classifier = classifier.cuda()
else:
    classifier = classifier.cpu()
    
classifier.eval()

ground_truth = []
predictions = []
for batch in dataloader:
    x_surnames = Variable(batch['x_surnames'])
    y_nationalities = Variable(batch['y_nationalities'])
    if args.cuda:
        x_surnames = x_surnames.cuda()
        y_nationalities = y_nationalities.cuda()
    
    outputs = classifier(x_surnames)
    
    _, y_preds = torch.max(outputs, dim=1)
    
    ground_truth.append(y_nationalities.data.cpu().numpy())
    predictions.append(y_preds.data.cpu().squeeze().numpy())
    
ground_truth = np.concatenate(ground_truth)
predictions = np.concatenate(predictions)

print(accuracy_score(ground_truth, predictions)) 
sns.factorplot(data=name_dataset_test.data, y='nationality', 
               kind='count', size=5, aspect=3);
plt.title("Counts per Nationality in the Surnames Test Set")
labels = name_dataset_test.data.nationality.values
sns.set_style('white')
sns.set_context('poster')
labels = [number_to_label[i] for i in range(len(number_to_label))]
g = sns.heatmap(confusion_matrix(ground_truth, predictions), cmap='Greens_r');
g.set_xticklabels(labels, rotation=30, size=11);
g.set_yticklabels(labels, rotation=0);<jupyter_output><empty_output><jupyter_text>Exercise: run your name through the model<jupyter_code>name = extract_name_features('mcmahan')
name = name.view(1,-1) # reshape because model expects a batch
if args.cuda:
    preds = classifier(Variable(name).cuda(), apply_softmax=True)
else:
    preds = classifier(Variable(name), apply_softmax=True)
top_pred, index = torch.max(preds, dim=1)
number_to_label[index.data.tolist()[0]]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12136767008874196, 'fraction_numerical': 0.010179223942926745, 'mean_word_length': 4.282628676470588, 'pattern_counts': {'":': 0, '<': 77, '<?xml version=': 0, '>': 78, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '186782', 'license': 'MIT', 'n_tokens_mistral': 4112, 'n_tokens_neox': 3937, 'n_words': 716, 'path': 'day_1/2_classify_names_with_MLP.ipynb', 'repo_name': 'joosthub/pytorch-nlp-tutorial-ny2018', 'size': '11502'}