<jupyter_start><jupyter_text>Get pre-trained embeddings<jupyter_code># encode the pretrained embedding to text file
model = KeyedVectors.load_word2vec_format('/home/hb1500/Plated/vocab.bin', binary=True)
model.save_word2vec_format('pretrained_embd.txt', binary=False)
fname = '../../data/glove.6B.50d.txt'
words_emb_dict = load_emb_vectors(fname)<jupyter_output><empty_output><jupyter_text>Load Cleaned Data<jupyter_code>steps = ['step_one','step_two', 'step_three', 'step_four', 'step_five', 'step_six']
steps_aug = ['step_one_sp', 'step_two_sp', 'step_three_sp',
             'step_four_sp', 'step_five_sp', 'step_six_sp']
tags = ['tag_cuisine_indian', 'tag_cuisine_nordic', 'tag_cuisine_european',
        'tag_cuisine_asian', 'tag_cuisine_mexican',
        'tag_cuisine_latin-american', 'tag_cuisine_french',
        'tag_cuisine_italian', 'tag_cuisine_african',
        'tag_cuisine_mediterranean', 'tag_cuisine_american',
        'tag_cuisine_middle-eastern']
data_with_aug = pd.read_csv('../data/recipe_data_with_aug.csv', index_col=0)
data_with_aug_tags = data_with_aug[steps+steps_aug+tags]
print(data_with_aug_tags.columns)<jupyter_output>Index(['step_one', 'step_two', 'step_three', 'step_four', 'step_five',
       'step_six', 'step_one_sp', 'step_two_sp', 'step_three_sp',
       'step_four_sp', 'step_five_sp', 'step_six_sp', 'tag_cuisine_indian',
       'tag_cuisine_nordic', 'tag_cuisine_european', 'tag_cuisine_asian',
       'tag_cuisine_mexican', 'tag_cuisine_latin-american',
       'tag_cuisine_french', 'tag_cuisine_italian', 'tag_cuisine_african',
       'tag_cuisine_mediterranean', 'tag_cuisine_american',
       'tag_cuisine_middle-eastern'],
      dtype='object')<jupyter_text>Tokenization<jupyter_code>print('Processing original instruction data')
# tokenize each steps on original datasets
steps_token = []
for step in steps:
    steps_token.append(step+'_token')
    data_with_aug_tags[step+'_token'] = tokenize_dataset(data_with_aug_tags[step])
    print(step, 'has been tokenized.')

# tokenize each steps on augmented datasets
print('Processing augmented instruction data')
steps_aug_token = []
for step in steps_aug:
    steps_aug_token.append(step+'_token')
    data_with_aug_tags[step+'_token'] = tokenize_dataset(data_with_aug_tags[step])
    print(step, 'has been tokenized.')
data_with_aug_tags = data_with_aug_tags[steps_token+steps_aug_token+tags]
data_with_aug_tags.columns<jupyter_output><empty_output><jupyter_text>Split train and test sets<jupyter_code>train, test_data = train_test_split(data_with_aug_tags, test_size=0.1, random_state=RANDOM_STATE)
test_data = test_data[steps_token+tags]
#train_data, val_data, train_tags, val_tags = train_test_split(X_train, y_train, test_size=0.1, random_state=RANDOM_STATE)
aug2ori_colname = dict(zip(steps_aug_token+tags, steps_token+tags))<jupyter_output><empty_output><jupyter_text>Cross validation for train and validation<jupyter_code>rnn_types = {
    'rnn': nn.RNN,
    'lstm': nn.LSTM,
    'gru': nn.GRU
}

params = dict(
    rnn1_type = 'gru',
    rnn2_type = 'gru',
    bi = True,
    hidden_dim1 = 30,
    hidden_dim2 = 30,
    num_classes = 1,
    
    num_epochs = 10,
    batch_size = 50,
    learning_rate = 0.01,
    step_max_descent = 3,
    
    add_data_aug = True,
    cuda_on = True
)
predicted_tags = 'tag_cuisine_american' 
kf = KFold(n_splits=5, shuffle=True, random_state=RANDOM_STATE)
k = 1
val_auc_kf = []
for train_index, val_index in kf.split(train):
    print('===================== This is the Kfold {} ====================='.format(k))
    k += 1
    val_data = train[steps_token+tags].iloc[val_index]
    train_data = train.iloc[train_index]
    
    if params['add_data_aug']:
        ##### add augmentation to training set by index #####
        train_org = train_data[steps_token+tags]
        train_aug = train_data[steps_aug_token+tags]
        train_aug.rename(index=str, columns=aug2ori_colname, inplace=True)
        # concatenate dfs
        train_data = pd.concat([train_org, train_aug], axis=0, ignore_index=False)
        ##### add augmentation to training set by index #####
    else:
        train_data = train_data[steps_token+tags]
    
    #print(len(train_data), len(train_data.dropna()))
    #look up
    train_targets = list(train_data[predicted_tags])
    val_targets = list(val_data[predicted_tags])
    test_targets = list(test_data[predicted_tags])
    
    train_X = train_data[steps_token]
    val_X = val_data[steps_token]
    test_X = test_data[steps_token]
    all_train_tokens = all_tokens_list(train_X)
    max_vocab_size = len(list(set(all_train_tokens)))
    token2id, id2token = build_vocab(all_train_tokens, max_vocab_size)
    emb_weight = build_emb_weight(words_emb_dict, id2token)
    train_data_indices = token2index_dataset(train_X, token2id)
    val_data_indices = token2index_dataset(val_X, token2id)
    test_data_indices = token2index_dataset(test_X, token2id)

    # batchify datasets: 
    batch_size = params['batch_size']
    max_sent_len = np.array([94, 86, 87, 90, 98, 91])
    train_loader, val_loader, test_loader = create_dataset_obj(train_data_indices, val_data_indices,
                                                           test_data_indices, train_targets,
                                                           val_targets, test_targets,
                                                           batch_size, max_sent_len, 
                                                           collate_func)
    
    val_auc, val_acc = model_train(params, emb_weight, train_loader, val_loader, test_loader)
    val_auc_kf.append(val_auc)
    
def model_train(params, emb_weight, train_loader, val_loader, test_loader):
    rnn1_type = params['rnn1_type'] 
    rnn_1 = rnn_types[rnn1_type]
    rnn2_type = params['rnn2_type']
    rnn_2 = rnn_types[rnn2_type]
    bi = params['bi']
    hidden_dim1 = params['hidden_dim1']
    hidden_dim2 = params['hidden_dim2']
    num_classes = params['num_classes']
    batch_size = params['batch_size']
    cuda_on = params['cuda_on']

    weights_matrix = torch.from_numpy(emb_weight)
    model = two_stage_RNN(rnn_1, hidden_dim1, bi, rnn_2, hidden_dim2, batch_size, 
                          cuda_on, weights_matrix, num_classes)
    model_parameters = filter(lambda p: p.requires_grad, model.parameters())
    print('The number of train parameters', sum([np.prod(p.size()) for p in model_parameters]))
    model = model.to(device)

    #parameter for training
    learning_rate = params['learning_rate']
    num_epochs = params['num_epochs'] # number epoch to train

    # Criterion and Optimizer
    criterion = nn.BCEWithLogitsLoss() #torch.nn.BCELoss(); torch.nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
    
    train_loss_list = []
    train_AUC_list = []
    val_AUC_list = []
    train_ACC_list = []
    val_ACC_list = []
    max_val_auc = 0
    step_max_descent = params['step_max_descent']
    for epoch in range(num_epochs):
        for i, (steps_batch, lengths_batch, labels_batch) in enumerate(train_loader):
            for step_id in range(6):
                lengths_batch[step_id] = lengths_batch[step_id].to(device)
                steps_batch[step_id] = steps_batch[step_id].to(device)
            model.train()
            optimizer.zero_grad()
            outputs = model(steps_batch, lengths_batch)
            loss = criterion(outputs, labels_batch.view(-1,1).float().to(device)) 
            train_loss_list.append(loss.item())
            loss.backward()
            optimizer.step()
            # validate every 10 step
            if i % 10 == 0:
                val_auc, val_acc = test_model(val_loader, model)
                print('{}/{}, Step:{}/{}, TrainLoss:{:.6f}, ValAUC:{:.6f} ValAcc:{:.6f}'.format(
                    epoch+1, num_epochs, i+1, len(train_loader), loss, val_auc, val_acc))
                train_auc, train_acc = test_model(train_loader, model)
                train_AUC_list.append(train_auc)
                val_AUC_list.append(val_auc)
                train_ACC_list.append(train_acc)
                val_ACC_list.append(val_acc)
                
                # early stop
                if max_val_auc < val_auc:
                    max_val_auc = val_auc
                    step_num_descent = 0
                else:
                    step_num_descent += 1
                if step_max_descent == step_num_descent:
                    print('early stop!')
                    break
        print('Epoch: [{}/{}], trainAUC: {:.6f}, trainAcc: {:.6f}'.format(epoch+1, num_epochs, train_auc, train_acc))
        print('Epoch: [{}/{}], ValAUC: {:.6f}, ValAcc: {:.6f}'.format(epoch+1, num_epochs, val_auc, val_acc))
        if step_max_descent == step_num_descent:
            break
    #return train_loss_list, train_AUC_list, val_AUC_list, train_ACC_list, val_ACC_list
    val_auc_mean = np.mean(val_AUC_list[-step_max_descent*2+1:])
    val_acc_mean = np.mean(val_ACC_list[-step_max_descent*2+1:])
    return val_auc_mean, val_acc_mean<jupyter_output><empty_output><jupyter_text>All tokens from training set<jupyter_code># form all tokens list
all_train_tokens = all_tokens_list(train_data)<jupyter_output><empty_output><jupyter_text>Let's decide which tag to predict for trail<jupyter_code>data_cuisine_tags.iloc[:,1:].sum()/data_cuisine_tags.iloc[:,1:].shape[0]<jupyter_output><empty_output><jupyter_text>Choose tag: tag_cuisine_american, which 27.3525% are 1   Build vocabulary and indexing<jupyter_code>len(list(set(all_train_tokens)))
token_counter = Counter(all_train_tokens)
# token_counter.most_common
emb_weight = build_emb_weight(words_emb_dict, id2token)
sum(np.sum(emb_weight,1)==0)/emb_weight.shape[0]<jupyter_output><empty_output><jupyter_text>Reconstruct data strcuture for datasets tag_cuisine_indian            0.023525  85% auctag_cuisine_nordic            0.000399tag_cuisine_european          0.012360tag_cuisine_asian             0.182217  98% auctag_cuisine_mexican           0.013557tag_cuisine_latin-american    0.094896  90% auctag_cuisine_french            0.077352  72% auctag_cuisine_italian           0.233254  80% auctag_cuisine_african           0.003987tag_cuisine_mediterranean     0.076555  88% auctag_cuisine_american          0.273525  80% auctag_cuisine_middle-eastern    0.046252  87% auc<jupyter_code># test model
logits_all = []
labels_all = []
model.eval()
for steps_batch, lengths_batch, labels_batch in test_loader:
    for step_id in range(6):
        lengths_batch[step_id] = lengths_batch[step_id].to(device)
        steps_batch[step_id] = steps_batch[step_id].to(devi) 
    logits = model(steps_batch, lengths_batch)
    logits_all.extend(list(logits.cpu().detach().numpy()))
    labels_all.extend(list(labels_batch.numpy()))
logits_all = np.array(logits_all)
labels_all = np.array(labels_all)
auc = roc_auc_score(labels_all, logits_all)
predicts = (logits_all > 0.5).astype(int)
acc = np.mean(predicts==labels_all)
auc
from sklearn import metrics
fpr, tpr, thresholds = metrics.roc_curve(labels_all, logits_all, pos_label=1)
plt.plot(fpr, tpr)
for p in model.parameters():
    if p.requires_grad:
        print(p.size())<jupyter_output>torch.Size([90, 50])
torch.Size([90, 30])
torch.Size([90])
torch.Size([90])
torch.Size([90, 50])
torch.Size([90, 30])
torch.Size([90])
torch.Size([90])
torch.Size([90, 60])
torch.Size([90, 30])
torch.Size([90])
torch.Size([90])
torch.Size([1, 30])
torch.Size([1])
{'fraction_non_alphanumeric': 0.11817547531833246, 'fraction_numerical': 0.02250130821559393, 'mean_word_length': 3.732562938505984, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '510789', 'license': 'Apache-2.0', 'n_tokens_mistral': 4205, 'n_tokens_neox': 4036, 'n_words': 647, 'path': 'src/.ipynb_checkpoints/Two_Stage_LSTM_cuisine_tags_CV_main-checkpoint.ipynb', 'repo_name': 'hb1500/Plated_Recipe_Tags_Predict', 'size': '11466'}