<commit_before>#!/usr/bin/python
import argparse
import collections
import os
import sys
import time

sys.path.insert(0, os.path.abspath(os.path.join(__file__, os.pardir, os.pardir)))

from scripts.engine import MMTServerApi
from scripts.libs import multithread

__author__ = '<NAME>'


def _sorted_features_list(features=None):
    if features is None:
        features = Api.get_features()

    l = [str(f) for f, _ in features.iteritems()]
    l.sort()
    return l


class _DocumentTranslator:
    def __init__(self, corpus, nbest, nbest_file, workers=100):
        self.corpus = corpus
        self.nbest = nbest
        self.nbest_file = nbest_file

        self.weights = None
        self.skip_context = False
        self._line_id = 0

        self._pool = multithread.Pool(workers)
        self._features = None

    def set_skipcontext(self, skip_context):
        self.skip_context = skip_context

    def set_weights(self, raw):
        if raw is None:
            self.weights = None
            return

        self.weights = {}
        array = []

        for token in raw.split():
            if token[-1:] == '=':
                self.weights[token.rstrip('=')] = array = []
            else:
                array.append(float(token))

    @staticmethod
    def _get_translation(line, nbest, session):
        return Api.translate(line, session=session, processing=False, nbest=nbest)

    def _print(self, translation, nbest_out):
        print translation['translation'].encode('utf-8')
        sys.stdout.flush()

        for hyp in translation['nbest']:
            scores = []

            for feature in self._features:
                if feature in hyp['scores']:
                    scores.append(feature + '=')
                    for s in hyp['scores'][feature]:
                        scores.append(str(s))

            nbest_out.write(str(self._line_id))
            nbest_out.write(' ||| ')
            nbest_out.write(hyp['translation'].encode('utf-8'))
            nbest_out.write(' ||| ')
            nbest_out.write(' '.join(scores))
            nbest_out.write(' ||| ')
            nbest_out.write(str(hyp['totalScore']))
            nbest_out.write('\n')

    def run(self):
        self._line_id = 0

        if self.weights is not None:
            Api.update_features(self.weights)
        time.sleep(1)

        self._features = _sorted_features_list()

        with open(self.nbest_file, 'ab') as nbest_out:
            with open(self.corpus) as source:
                for line in source:
                    tokenized, original = line.strip().split(':')

                    session = None

                    if not self.skip_context:
                        context = Api.get_context_f(original)
                        session = Api.create_session(context)['id']

                    with open(tokenized) as doc:
                        jobs = []
                        for docline in doc:
                            result = self._pool.apply_async(self._get_translation, (docline, self.nbest, session))
                            jobs.append(result)

                        for job in jobs:
                            translation = job.get()
                            self._print(translation, nbest_out)
                            self._line_id += 1

                    if session is not None:
                        Api.close_session(session)

        self._pool.terminate()


def show_weighs():
    features_map = Api.get_features()
    features = _sorted_features_list(features_map)

    for feature in features:<commit_msg>fix to terminate mert in case of translation errors<commit_after>#!/usr/bin/python
import argparse
import collections
import os
import sys
import time

sys.path.insert(0, os.path.abspath(os.path.join(__file__, os.pardir, os.pardir)))

from scripts.engine import MMTServerApi
from scripts.libs import multithread

__author__ = '<NAME>'


def _sorted_features_list(features=None):
    if features is None:
        features = Api.get_features()

    l = [str(f) for f, _ in features.iteritems()]
    l.sort()
    return l


class _DocumentTranslator:
    def __init__(self, corpus, nbest, nbest_file, workers=100):
        self.corpus = corpus
        self.nbest = nbest
        self.nbest_file = nbest_file

        self.weights = None
        self.skip_context = False
        self._line_id = 0

        self._pool = multithread.Pool(workers)
        self._features = None

    def set_skipcontext(self, skip_context):
        self.skip_context = skip_context

    def set_weights(self, raw):
        if raw is None:
            self.weights = None
            return

        self.weights = {}
        array = []

        for token in raw.split():
            if token[-1:] == '=':
                self.weights[token.rstrip('=')] = array = []
            else:
                array.append(float(token))

    @staticmethod
    def _get_translation(line, nbest, session):
        return Api.translate(line, session=session, processing=False, nbest=nbest)

    def _print(self, translation, nbest_out):
        print translation['translation'].encode('utf-8')
        sys.stdout.flush()

        for hyp in translation['nbest']:
            scores = []

            for feature in self._features:
                if feature in hyp['scores']:
                    scores.append(feature + '=')
                    for s in hyp['scores'][feature]:
                        scores.append(str(s))

            nbest_out.write(str(self._line_id))
            nbest_out.write(' ||| ')
            nbest_out.write(hyp['translation'].encode('utf-8'))
            nbest_out.write(' ||| ')
            nbest_out.write(' '.join(scores))
            nbest_out.write(' ||| ')
            nbest_out.write(str(hyp['totalScore']))
            nbest_out.write('\n')

    def run(self):
        self._line_id = 0

        if self.weights is not None:
            Api.update_features(self.weights)
        time.sleep(1)

        self._features = _sorted_features_list()

        try:
            with open(self.nbest_file, 'ab') as nbest_out:
                with open(self.corpus) as source:
                    for line in source:
                        tokenized, original = line.strip().split(':')

                        session = None

                        if not self.skip_context:
                            context = Api.get_context_f(original)
                            session = Api.create_session(context)['id']

                        with open(tokenized) as doc:
                            jobs = []
                            for docline in doc:
                                result = self._pool.apply_async(self._get_translation, (docline, self.nbest, session))
                                jobs.append(result)

                            for job in jobs:
                                translation = job.get()
                                self._print(translation, nbest_out)
                                self._line_id += 1

                        if session is not None:
                            Api.close_session(session)
        finally:
            self._pool.terminate()


def show_weighs():
    features_map = Api.get_features()
    features = _sorted_features_list(features_map)

    for feature in features:
{'fraction_non_alphanumeric': 0.10323637853338796, 'fraction_numerical': 0.0030042332377440938, 'mean_word_length': 1.6555474981870921, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3186267', 'n_tokens_mistral': 2094, 'n_tokens_neox': 1986, 'n_words': 445}