# Path: lxmls/data.py
# def find(filename):


from lxmls import data
import readers.pos_corpus as pcc
import sequences.extended_feature as exfc
import sequences.structured_perceptron as spc
MAX_NR_SENTENCES = 100000
MODEL_DIR = "../models/wsj_postag/"


def build_corpus_features():
    corpus = pcc.PostagCorpus()
    train_seq = corpus.read_sequence_list_conll(data.find('train-02-21.conll'),
        max_sent_len=MAX_SENT_SIZE,
        max_nr_sent=MAX_NR_SENTENCES)
    corpus.add_sequence_list(train_seq)
    features = exfc.ExtendedFeatures(corpus)
    features.build_features()
    corpus.save_corpus(MODEL_DIR)
    features.save_features(MODEL_DIR + "features.txt")
    return corpus, features


def train_pos(corpus, features):
    model = spc.StructuredPercetron(corpus, features)
    model.nr_rounds = 10
    model.train_supervised(corpus.sequence_list.seq_list)
    model.save_model(MODEL_DIR)
    return model


def eval_model(corpus, features, model):
    dev_seq = corpus.read_sequence_list_conll(data.find('dev-22.conll'))
    pred_dev = model.viterbi_decode_corpus_log(dev_seq.seq_list)
    eval_dev = model.evaluate_corpus(dev_seq.seq_list, pred_dev)
    print("Accuracy on wsj development %f" % eval_dev)

# Path: lxmls/data.py
# def find(filename):


# Path: lxmls/pos_tagging/train_pos_tag.py
from lxmls import data
import readers.pos_corpus as pcc
import sequences.extended_feature as exfc
import sequences.structured_perceptron as spc

MAX_NR_SENTENCES = 100000
MODEL_DIR = "../models/wsj_postag/"


def build_corpus_features():
    corpus = pcc.PostagCorpus()
    train_seq = corpus.read_sequence_list_conll(data.find('train-02-21.conll'),
        max_sent_len=MAX_SENT_SIZE,
        max_nr_sent=MAX_NR_SENTENCES)
    corpus.add_sequence_list(train_seq)
    features = exfc.ExtendedFeatures(corpus)
    features.build_features()
    corpus.save_corpus(MODEL_DIR)
    features.save_features(MODEL_DIR + "features.txt")
    return corpus, features


def train_pos(corpus, features):
    model = spc.StructuredPercetron(corpus, features)
    model.nr_rounds = 10
    model.train_supervised(corpus.sequence_list.seq_list)
    model.save_model(MODEL_DIR)
    return model


def eval_model(corpus, features, model):
    dev_seq = corpus.read_sequence_list_conll(data.find('dev-22.conll'))
    pred_dev = model.viterbi_decode_corpus_log(dev_seq.seq_list)
    eval_dev = model.evaluate_corpus(dev_seq.seq_list, pred_dev)
    print("Accuracy on wsj development %f" % eval_dev)

    test_seq = corpus.read_sequence_list_conll(data.find('test-23.conll'))