<jupyter_start><jupyter_text>please use python 3<jupyter_code>%time
%load_ext autotime
%load_ext autoreload
%autoreload 2

import torch
from torch import nn
from torch.autograd import Variable

from data_loader import DataLoader
from model import UniSkip
from config import *
from datetime import datetime, timedelta<jupyter_output>CPU times: user 4 ms, sys: 0 ns, total: 4 ms
Wall time: 4.29 µs<jupyter_text>train a new model<jupyter_code>d = DataLoader('../dir_HugeFiles/instructions/skip_instruction.csv')
mod = UniSkip()
if USE_CUDA:
    mod.cuda(CUDA_DEVICE)
lr = 3e-4
optimizer = torch.optim.Adam(params=mod.parameters(), lr=lr)
loss_trail = []
last_best_loss = None
current_time = datetime.utcnow()

def debug(i, loss, prev, nex, prev_pred, next_pred):
    global loss_trail
    global last_best_loss
    global current_time

    this_loss = loss.item()
    loss_trail.append(this_loss)
    loss_trail = loss_trail[-20:]
    new_current_time = datetime.utcnow()
    time_elapsed = str(new_current_time - current_time)
    current_time = new_current_time
    print("Iteration {}: time = {} last_best_loss = {}, this_loss = {}".format(
              i, time_elapsed, last_best_loss, this_loss))
    
    '''
    print("prev = {}\nnext = {}\npred_prev = {}\npred_next = {}".format(
        d.convert_indices_to_sentences(prev),
        d.convert_indices_to_sentences(nex),
        d.convert_indices_to_sentences(prev_pred),
        d.convert_indices_to_sentences(next_pred),
    ))
    '''

    try:
        trail_loss = sum(loss_trail)/len(loss_trail)
        if last_best_loss is None or last_best_loss > trail_loss:
            print("Loss improved from {} to {}".format(last_best_loss, trail_loss))
            
            save_loc = "./saved_models/skip-best".format(lr, VOCAB_SIZE)
            print("saving model at {}".format(save_loc))
            torch.save(mod.state_dict(), save_loc)
            
            last_best_loss = trail_loss
    except Exception as e:
        print("Couldn't save model because {}".format(e))

print("Starting training...")
print('current GPU is on %d ' %(CUDA_DEVICE)) # should change at the config.py
# a million iterations
for i in range(0, 1000000):
    sentences, lengths = d.fetch_batch(32 * 8) # remember to change cuda device

    loss, prev, nex, prev_pred, next_pred  = mod(sentences, lengths)
    if i % 100 == 0:
        debug(i, loss, prev, nex, prev_pred, next_pred)

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()<jupyter_output><empty_output><jupyter_text>resume the training<jupyter_code># model to resume
loc = "./prev_model/skip-best"

d = DataLoader('../dir_HugeFiles/instructions/skip_instruction.csv')
mod = UniSkip()
mod.load_state_dict(torch.load(loc, map_location=lambda storage, loc: storage))

if USE_CUDA:
    mod.cuda(CUDA_DEVICE)
lr = 3e-4
optimizer = torch.optim.Adam(params=mod.parameters(), lr=lr)
loss_trail = []
last_best_loss = None
current_time = datetime.utcnow()

def debug(i, loss, prev, nex, prev_pred, next_pred):
    global loss_trail
    global last_best_loss
    global current_time

    this_loss = loss.item()
    loss_trail.append(this_loss)
    loss_trail = loss_trail[-20:]
    new_current_time = datetime.utcnow()
    time_elapsed = str(new_current_time - current_time)
    current_time = new_current_time
    print("Iteration {}: time = {} last_best_loss = {}, this_loss = {}".format(
              i, time_elapsed, last_best_loss, this_loss))
    
    '''
    print("prev = {}\nnext = {}\npred_prev = {}\npred_next = {}".format(
        d.convert_indices_to_sentences(prev),
        d.convert_indices_to_sentences(nex),
        d.convert_indices_to_sentences(prev_pred),
        d.convert_indices_to_sentences(next_pred),
    ))
    '''

    try:
        trail_loss = sum(loss_trail)/len(loss_trail)
        if last_best_loss is None or last_best_loss > trail_loss:
            print("Loss improved from {} to {}".format(last_best_loss, trail_loss))
            save_loc = "./prev_model/" + "skip-best-loss{0:.3f}".format(trail_loss)
            print("saving model at {}".format(save_loc))
            torch.save(mod.state_dict(), save_loc)
            
            last_best_loss = trail_loss
    except Exception as e:
        print("Couldn't save model because {}".format(e))

print("Starting training...")
print('current GPU is on %d ' %(CUDA_DEVICE)) # should change at the config.py
# a million iterations
for i in range(0, 1000000):
    sentences, lengths = d.fetch_batch(32 * 8) # remember to change cuda device

    loss, prev, nex, prev_pred, next_pred  = mod(sentences, lengths)
    if i % 100 == 0:
        debug(i, loss, prev, nex, prev_pred, next_pred)

    optimizer.zero_grad()
    loss.backward()
    optimizer.step()<jupyter_output>Loading text file at ../dir_HugeFiles/instructions/skip_instruction.csv
Making dictionary for these words
Using cached dictionary at ../dir_HugeFiles/instructions/skip_instruction.csv.pkl
Making reverse dictionary
Starting training...
current GPU is on 3 
Iteration 0: time = 0:00:00.370761 last_best_loss = None, this_loss = 10.23737621307373
Loss improved from None to 10.23737621307373
saving model at ./prev_model/skip-best-loss10.237
Iteration 100: time = 0:00:57.285677 last_best_loss = 10.23737621307373, this_loss = 10.156049728393555
Loss improved from 10.23737621307373 to 10.196712970733643
saving model at ./prev_model/skip-best-loss10.197
Iteration 200: time = 0:00:57.271184 last_best_loss = 10.196712970733643, this_loss = 10.151169776916504
Loss improved from 10.196712970733643 to 10.18153190612793
saving model at ./prev_model/skip-best-loss10.182
Iteration 300: time = 0:01:08.380468 last_best_loss = 10.18153190612793, this_loss = 10.3784761428833
Iteration 400: time = 0:01:04.08[...]
{'fraction_non_alphanumeric': 0.11307359307359308, 'fraction_numerical': 0.05610389610389611, 'mean_word_length': 3.7112561174551386, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '582880', 'license': 'MIT', 'n_tokens_mistral': 2215, 'n_tokens_neox': 1954, 'n_words': 446, 'path': 'Train-Copy1.ipynb', 'repo_name': 'helenalee1994/skip-thoughts', 'size': '5775'}