def train(data):
    print('Training model...')
    data.show_data_summary()
    save_data_name = (data.model_dir + '.dset')
    data.save(save_data_name)
    if data.sentence_classification:
        model = SentClassifier(data)
    else:
        model = SeqLabel(data)
    if (data.optimizer.lower() == 'sgd'):
        optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2)
    elif (data.optimizer.lower() == 'adagrad'):
        optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2)
    elif (data.optimizer.lower() == 'adadelta'):
        optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2)
    elif (data.optimizer.lower() == 'rmsprop'):
        optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2)
    elif (data.optimizer.lower() == 'adam'):
        optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2)
    else:
        print(('Optimizer illegal: %s' % data.optimizer))
        exit(1)
    best_dev = (- 10)
    for idx in range(data.HP_iteration):
        epoch_start = time.time()
        temp_start = epoch_start
        print(('Epoch: %s/%s' % (idx, data.HP_iteration)))
        if (data.optimizer == 'SGD'):
            optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr)
        instance_count = 0
        sample_id = 0
        sample_loss = 0
        total_loss = 0
        right_token = 0
        whole_token = 0
        random.shuffle(data.train_Ids)
        print('Shuffle: first input word list:', data.train_Ids[0][0])
        model.train()
        model.zero_grad()
        batch_size = data.HP_batch_size
        batch_id = 0
        train_num = len(data.train_Ids)
        total_batch = ((train_num // batch_size) + 1)
        for batch_id in range(total_batch):
            start = (batch_id * batch_size)
            end = ((batch_id + 1) * batch_size)
            if (end > train_num):
                end = train_num
            instance = data.train_Ids[start:end]
            if (not instance):
                continue
            (batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask) = batchify_with_label(instance, data.HP_gpu, True, data.sentence_classification)
            instance_count += 1
            (loss, tag_seq) = model.calculate_loss(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask)
            (right, whole) = predict_check(tag_seq, batch_label, mask, data.sentence_classification)
            right_token += right
            whole_token += whole
            sample_loss += loss.item()
            total_loss += loss.item()
            if ((end % 500) == 0):
                temp_time = time.time()
                temp_cost = (temp_time - temp_start)
                temp_start = temp_time
                print(('     Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f' % (end, temp_cost, sample_loss, right_token, whole_token, ((right_token + 0.0) / whole_token))))
                if ((sample_loss > 100000000.0) or (str(sample_loss) == 'nan')):
                    print('ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....')
                    exit(1)
                sys.stdout.flush()
                sample_loss = 0
            loss.backward()
            optimizer.step()
            model.zero_grad()
        temp_time = time.time()
        temp_cost = (temp_time - temp_start)
        print(('     Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f' % (end, temp_cost, sample_loss, right_token, whole_token, ((right_token + 0.0) / whole_token))))
        epoch_finish = time.time()
        epoch_cost = (epoch_finish - epoch_start)
        print(('Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s,  total loss: %s' % (idx, epoch_cost, (train_num / epoch_cost), total_loss)))
        print('totalloss:', total_loss)
        if ((total_loss > 100000000.0) or (str(total_loss) == 'nan')):
            print('ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....')
            exit(1)
        (speed, acc, p, r, f, _, _, sensitivities) = evaluate(data, model, 'dev')
        dev_finish = time.time()
        dev_cost = (dev_finish - epoch_finish)
        if data.seg:
            current_score = f
            print(('Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f' % (dev_cost, speed, acc, p, r, f)))
        else:
            current_score = acc
            print(('Dev: time: %.2fs speed: %.2fst/s; acc: %.4f' % (dev_cost, speed, acc)))
        if (current_score > best_dev):
            if data.seg:
                print('Exceed previous best f score:', best_dev)
            else:
                print('Exceed previous best acc score:', best_dev)
            model_name = (((data.model_dir + '.') + str(idx)) + '.model')
            print('Save current best model in file:', model_name)
            torch.save(model.state_dict(), model_name)
            best_dev = current_score
        (speed, acc, p, r, f, _, _, sensitivities) = evaluate(data, model, 'test')
        test_finish = time.time()
        test_cost = (test_finish - dev_finish)
        if data.seg:
            print(('Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f' % (test_cost, speed, acc, p, r, f)))
        else:
            print(('Test: time: %.2fs, speed: %.2fst/s; acc: %.4f' % (test_cost, speed, acc)))
        gc.collect()
def train(data):
    print('Training model...')
    data.show_data_summary()
    save_data_name = (data.model_dir + '.dset')
    data.save(save_data_name)
    if data.sentence_classification:
        model = SentClassifier(data)
    else:
        model = SeqLabel(data)
    if (data.optimizer.lower() == 'sgd'):
        optimizer = optim.SGD(model.parameters(), lr=data.HP_lr, momentum=data.HP_momentum, weight_decay=data.HP_l2)
    elif (data.optimizer.lower() == 'adagrad'):
        optimizer = optim.Adagrad(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2)
    elif (data.optimizer.lower() == 'adadelta'):
        optimizer = optim.Adadelta(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2)
    elif (data.optimizer.lower() == 'rmsprop'):
        optimizer = optim.RMSprop(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2)
    elif (data.optimizer.lower() == 'adam'):
        optimizer = optim.Adam(model.parameters(), lr=data.HP_lr, weight_decay=data.HP_l2)
    else:
        print(('Optimizer illegal: %s' % data.optimizer))
        exit(1)
    best_dev = (- 10)
    for idx in range(data.HP_iteration):
        epoch_start = time.time()
        temp_start = epoch_start
        print(('Epoch: %s/%s' % (idx, data.HP_iteration)))
        if (data.optimizer == 'SGD'):
            optimizer = lr_decay(optimizer, idx, data.HP_lr_decay, data.HP_lr)
        instance_count = 0
        sample_id = 0
        sample_loss = 0
        total_loss = 0
        right_token = 0
        whole_token = 0
        random.shuffle(data.train_Ids)
        print('Shuffle: first input word list:', data.train_Ids[0][0])
        model.train()
        model.zero_grad()
        batch_size = data.HP_batch_size
        batch_id = 0
        train_num = len(data.train_Ids)
        total_batch = ((train_num // batch_size) + 1)
        for batch_id in range(total_batch):
            start = (batch_id * batch_size)
            end = ((batch_id + 1) * batch_size)
            if (end > train_num):
                end = train_num
            instance = data.train_Ids[start:end]
            if (not instance):
                continue
            (batch_word, batch_features, batch_wordlen, batch_wordrecover, batch_char, batch_charlen, batch_charrecover, batch_label, mask) = batchify_with_label(instance, data.HP_gpu, True, data.sentence_classification)
            instance_count += 1
            (loss, tag_seq) = model.calculate_loss(batch_word, batch_features, batch_wordlen, batch_char, batch_charlen, batch_charrecover, batch_label, mask)
            (right, whole) = predict_check(tag_seq, batch_label, mask, data.sentence_classification)
            right_token += right
            whole_token += whole
            sample_loss += loss.item()
            total_loss += loss.item()
            if ((end % 500) == 0):
                temp_time = time.time()
                temp_cost = (temp_time - temp_start)
                temp_start = temp_time
                print(('     Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f' % (end, temp_cost, sample_loss, right_token, whole_token, ((right_token + 0.0) / whole_token))))
                if ((sample_loss > 100000000.0) or (str(sample_loss) == 'nan')):
                    print('ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....')
                    exit(1)
                sys.stdout.flush()
                sample_loss = 0
            loss.backward()
            optimizer.step()
            model.zero_grad()
        temp_time = time.time()
        temp_cost = (temp_time - temp_start)
        print(('     Instance: %s; Time: %.2fs; loss: %.4f; acc: %s/%s=%.4f' % (end, temp_cost, sample_loss, right_token, whole_token, ((right_token + 0.0) / whole_token))))
        epoch_finish = time.time()
        epoch_cost = (epoch_finish - epoch_start)
        print(('Epoch: %s training finished. Time: %.2fs, speed: %.2fst/s,  total loss: %s' % (idx, epoch_cost, (train_num / epoch_cost), total_loss)))
        print('totalloss:', total_loss)
        if ((total_loss > 100000000.0) or (str(total_loss) == 'nan')):
            print('ERROR: LOSS EXPLOSION (>1e8) ! PLEASE SET PROPER PARAMETERS AND STRUCTURE! EXIT....')
            exit(1)
        (speed, acc, p, r, f, _, _, sensitivities) = evaluate(data, model, 'dev')
        dev_finish = time.time()
        dev_cost = (dev_finish - epoch_finish)
        if data.seg:
            current_score = f
            print(('Dev: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f' % (dev_cost, speed, acc, p, r, f)))
        else:
            current_score = acc
            print(('Dev: time: %.2fs speed: %.2fst/s; acc: %.4f' % (dev_cost, speed, acc)))
        if (current_score > best_dev):
            if data.seg:
                print('Exceed previous best f score:', best_dev)
            else:
                print('Exceed previous best acc score:', best_dev)
            model_name = (((data.model_dir + '.') + str(idx)) + '.model')
            print('Save current best model in file:', model_name)
            torch.save(model.state_dict(), model_name)
            best_dev = current_score
        (speed, acc, p, r, f, _, _, sensitivities) = evaluate(data, model, 'test')
        test_finish = time.time()
        test_cost = (test_finish - dev_finish)
        if data.seg:
            print(('Test: time: %.2fs, speed: %.2fst/s; acc: %.4f, p: %.4f, r: %.4f, f: %.4f' % (test_cost, speed, acc, p, r, f)))
        else:
            print(('Test: time: %.2fs, speed: %.2fst/s; acc: %.4f' % (test_cost, speed, acc)))
        gc.collect()