<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import argparse
import itertools as it
import operator as op
import pickle
import sys
from mmb_perceptron.dict_impl import CombinatorialPerceptron
from mmb_perceptron.feature_extractor import \
     Honnibal, Ratnaparkhi, Char
from mmb_perceptron.helper.pos_tagging import \
     log, check_counts_for_mode, extract_sentences, preprocess_sentences

def get_feature_extractor(name, context_size):
    if name == 'Honnibal':
        feature = Honnibal(sequenced=True)
    elif name == 'Ratnaparkhi':
        feature = Ratnaparkhi(sequenced=True)
    elif name == 'Char':
        feature = Char(sequenced=True)
    else:
        raise NotImplementedError(name + " is not implemented")
    feature.context_size = (context_size, context_size)
    return feature

def main():
    log("Reading input data...")
    (sentences, gold_tags, token_count, tag_count) = \
        extract_sentences(args.infile, encoding=args.enc)
    log("Parsed {0} token(s) with {1} tags in {2} sentence(s)."
        .format(token_count, tag_count, len(sentences)))
    check_counts_for_mode(token_count, tag_count, args.train)

    if args.preprocessing:
        sentences = preprocess_sentences(sentences)

    if args.train:
        log("Training...")
        model = CombinatorialPerceptron(
            averaged=args.averaging,
            iterations=args.iterations,
            learning_rate=1,
            sequenced=True,
            feature_extractor=get_feature_extractor(args.feature, args.context_size),
            log_to=sys.stderr
            )
        model.train(sentences, gold_tags)
        log("Saving...")
        with open(args.par, 'w') as f:
            pickle.dump(model, f)

    if not args.train:
        log("Loading model...")
        with open(args.par, 'r') as f:
            model = pickle.load(f)
            model.log_to = sys.stderr
        log("Tagging...")
        correct_count = 0
        predictions = model.predict_all(sentences)
        for sentence in it.izip(sentences, predictions, gold_tags):
            for (word, pred_tag, gold_tag) in it.izip(*sentence):
                print("{0}\t{1}".format(word, tag).encode("utf-8"))
                if gold_tag is not None and gold_tag == pred_tag:
                    correct_count += 1
            print() # line break between sentences
        if tag_count > 0:  # print evaluation
            log("Accuracy:  {0:7}/{1:7} correct ({2:.2f}%)"
                .format(correct_count, tag_count, (correct_count/tag_count)*100))

    log("Done.")


if __name__ == '__main__':
    description = "Part-of-speech tagger using a greedy perceptron algorithm."
    epilog = ("Input is expected to contain one token per line, with an empty "
              "line being treated as a sentence boundary.  Tokens and "
              "gold-standard POS tags (required for training, optional for "
              "tagging) must be separated by a <tab>.")
    parser = argparse.ArgumentParser(description=description, epilog=epilog)
    parser.add_argument('infile',
                        metavar='INPUT',<commit_msg>Fix minor bugs<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import argparse
import itertools as it
import operator as op
import pickle
import sys
from mmb_perceptron.dict_impl import CombinatorialPerceptron
from mmb_perceptron.feature_extractor import \
     Honnibal, Ratnaparkhi, Char
from mmb_perceptron.helper.pos_tagging import \
     log, check_counts_for_mode, extract_sentences, preprocess_sentences

def get_feature_extractor(name, context_size):
    if name == 'Honnibal':
        feature = Honnibal(sequenced=True)
    elif name == 'Ratnaparkhi':
        feature = Ratnaparkhi(sequenced=True)
    elif name == 'Char':
        feature = Char(sequenced=True)
    else:
        raise NotImplementedError(name + " is not implemented")
    feature.context_size = (context_size, context_size)
    return feature

def main():
    log("Reading input data...")
    (sentences, gold_tags, token_count, tag_count) = \
        extract_sentences(args.infile, encoding=args.enc)
    log("Parsed {0} token(s) with {1} tags in {2} sentence(s)."
        .format(token_count, tag_count, len(sentences)))
    check_counts_for_mode(token_count, tag_count, args.train)

    if args.preprocessing:
        sentences = preprocess_sentences(sentences)

    if args.train:
        log("Training...")
        model = CombinatorialPerceptron(
            averaged=args.averaging,
            iterations=args.iterations,
            learning_rate=1,
            sequenced=True,
            feature_extractor=get_feature_extractor(args.feature, args.context_size),
            log_to=sys.stderr
            )
        model.train(sentences, gold_tags)
        log("Saving...")
        with open(args.par, 'w') as f:
            pickle.dump(model, f)

    if not args.train:
        log("Loading model...")
        with open(args.par, 'r') as f:
            model = pickle.load(f)
            model.log_to = sys.stderr
        log("Tagging...")
        correct_count = 0
        predictions = model.predict_all(sentences)
        for sentence in it.izip(sentences, predictions, gold_tags):
            for (word, pred_tag, gold_tag) in it.izip(*sentence):
                print(u"{0}\t{1}".format(word, pred_tag).encode("utf-8"))
                if gold_tag is not None and gold_tag == pred_tag:
                    correct_count += 1
            print() # line break between sentences
        if tag_count > 0:  # print evaluation
            log("Accuracy:  {0:7}/{1:7} correct ({2:.2f}%)"
                .format(correct_count, tag_count, (float(correct_count)/tag_count)*100))

    log("Done.")


if __name__ == '__main__':
    description = "Part-of-speech tagger using a greedy perceptron algorithm."
    epilog = ("Input is expected to contain one token per line, with an empty "
              "line being treated as a sentence boundary.  Tokens and "
              "gold-standard POS tags (required for training, optional for "
              "tagging) must be separated by a <tab>.")
    parser = argparse.ArgumentParser(description=description, epilog=epilog)
    parser.add_argument('infile',
                        metavar='INPUT',
{'fraction_non_alphanumeric': 0.1084106172049888, 'fraction_numerical': 0.0063959066197633516, 'mean_word_length': 2.5886402753872635, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2231475', 'n_tokens_mistral': 1951, 'n_tokens_neox': 1874, 'n_words': 459}