<commit_before>import Queue
import sys
import threading
from xml.etree import ElementTree

import requests

from cli import IllegalArgumentException, IllegalStateException
from cli.libs import multithread
from cli.mmt.cluster import ClusterNode

__author__ = '<NAME>'


class Translator:
    def __init__(self, node, context_string=None, context_file=None, context_vector=None,
                 print_nbest=None, nbest_file=None):
        self.source_lang = node.engine.source_lang
        self.target_lang = node.engine.target_lang
        self._priority = ClusterNode.Api.PRIORITY_BACKGROUND
        self._api = node.api
        self._print_nbest = print_nbest

        self._nbest_out = None
        self._nbest_file = None

        if print_nbest:
            self._nbest_out = sys.stdout

            if nbest_file is not None:
                self._nbest_out = self._nbest_file = open(nbest_file, 'wb')

        self._context = None

        if context_string is not None:
            self._context = self._api.get_context_s(self.source_lang, self.target_lang, context_string)
        elif context_file is not None:
            self._context = self._api.get_context_f(self.source_lang, self.target_lang, context_file)
        elif context_vector is not None:
            self._context = self.__parse_context_vector(context_vector)

    @staticmethod
    def __parse_context_vector(text):
        context = []

        try:
            for score in text.split(','):
                id, value = score.split(':', 2)
                value = float(value)

                context.append({
                    'memory': int(id),
                    'score': value
                })
        except ValueError:
            raise IllegalArgumentException('invalid context weights map: ' + text)

        return context

    @staticmethod
    def _encode_translation(translation):
        return translation['translation'].encode('utf-8')

    @staticmethod
    def _encode_nbest(nbest):
        scores = []

        for name, values in nbest['scores'].iteritems():
            scores.append(name + '= ' + ' '.join([str(f) for f in values]))

        return [
            nbest['translation'],
            ' '.join(scores),
            str(nbest['totalScore'])
        ]

    def _translate(self, line, _=None):
        return self._api.translate(self.source_lang, self.target_lang, line, context=self._context,
                                   nbest=self._print_nbest, priority=self._priority)

    def execute(self, line):
        pass

    def flush(self):
        pass

    def close(self):
        if self._nbest_file is not None:
            self._nbest_file.close()


class BatchTranslator(Translator):
    def __init__(self, node, context_string=None, context_file=None, context_vector=None,
                 print_nbest=False, nbest_file=None, pool_size=100):
        Translator.__init__(self, node, context_string, context_file, context_vector, print_nbest, nbest_file)
        self._pool = multithread.Pool(pool_size)
        self._jobs = Queue.Queue(pool_size)
        self._line_id = 0
        self._printer_thread = threading.Thread(target=self._threaded_print)

        self._printer_thread.start()

    def execute(self, line):
        result = self._pool.apply_async(self._translate, (line, None))
        self._jobs.put(result, block=True)

    def flush(self):
        self._jobs.put(None, block=True)

    def _threaded_print(self):
        while True:
            job = self._jobs.get(block=True)

            if job is None:
                break

            translation = job.get()

            print self._encode_translation(translation)
            if self._print_nbest is not None:
                for nbest in translation['nbest']:
                    parts = [str(self._line_id)] + self._encode_nbest(nbest)
                    self._nbest_out.write((u' ||| '.join(parts)).encode('utf-8'))
                    self._nbest_out.write('\n')

            self._line_id += 1

    def close(self):
        self._printer_thread.join()
        Translator.close(self)
        self._pool.terminate()


class XLIFFTranslator(Translator):
    DEFAULT_NAMESPACE = 'urn:oasis:names:tc:xliff:document:1.2'
    SDL_NAMESPACE = 'http://sdl.com/FileTypes/SdlXliff/1.0'

    def __init__(self, node, context_string=None, context_file=None, context_vector=None):
        Translator.__init__(self, node, context_string, context_file, context_vector)
        self._content = []
        self._pool = multithread.Pool(100)

        ElementTree.register_namespace('', self.DEFAULT_NAMESPACE)
        ElementTree.register_namespace('sdl', self.SDL_NAMESPACE)
        ElementTree.register_namespace('mq', 'MQXliff')

    def execute(self, line):
        self._content.append(line)

    def _translate_transunit(self, tu, _=None):

        # if there is a sdl match to 
        if self._has_sdl_match(tu):
            return None

        source_tag = tu.find('{' + self.DEFAULT_NAMESPACE + '}source')
        seg_source_tag = tu.find('{' + self.DEFAULT_NAMESPACE + '}seg-source')
        target_tag = tu.find('{' + self.DEFAULT_NAMESPACE + '}target')

        # if this XLIFF file uses <seg-source> nodes, find its <mrk> children that have an ID and that are segments,
        # translate them and put the translation in the corresponding <mrk> children in <target>
        if seg_source_tag is not None:
            source_mrk_tags = seg_source_tag.findall('.//{' + self.DEFAULT_NAMESPACE + '}mrk[@mid]')

            # for each <mrk> element, find the corresponding <mrk> in <target>
            for source_mrk_tag in source_mrk_tags:
                mrk_tag_id = str(source_mrk_tag.attrib["mid"])
                target_mrk_tag = target_tag.find('.//{' + self.DEFAULT_NAMESPACE + '}mrk[@mid="' + mrk_tag_id + '"]')

                # if no <mrk> node with the right type and ID exists in <target>, raise an exception
                if target_mrk_tag is None:
                    raise IllegalArgumentException(
                        "<seg-source><mrk> tag with mid %s has no corresponding <target><mrk> tag" % mrk_tag_id)

                # extract the text to translate:

                # if there are more <mrk> descendants under this <mrk>, ignore this <mrk>;
                # the <mrk> descendants will be handled one by one by following iterations
                if len(source_mrk_tag.findall('.//{' + self.DEFAULT_NAMESPACE + '}mrk')) > 0:
                    continue

                # get the content of the mrk tag to translate, translate it,
                # parse the translation result as an XML element and append it under the target mrk tag
                source_text = self._get_string_content_of(source_mrk_tag)
                if source_text:
                    translation = self._translate(source_text)
                    self._append_xmlstring_to(translation['translation'], target_mrk_tag)

        # else just use the <source> and <target> nodes directly
        else:
            if source_tag is not None and target_tag is not None and source_tag.text:
                translation = self._translate(source_tag.text)
                target_tag.text = translation['translation']

        return None

    # This method checks if the passed trans-unit has an SDL match content and must therefore be skipped
    def _has_sdl_match(self, tu):
        match = tu.find('.//{' + self.SDL_NAMESPACE + '}seg[@percent="100"]')
        return True if match is not None else False

    # This method takes an ElementTree element and gets its content (text + subelements) as a string
    def _get_string_content_of(self, element):
        s = element.text or ""
        for sub_element in element:
            s += ElementTree.tostring(sub_element).replace('xmlns="' + self.DEFAULT_NAMESPACE + '" ', '')
        s += element.tail or ""
        return s

    # This method parses an xml string and appends its elements and text to the passed XML Element
    def _append_xmlstring_to(self, string, element_to_fill):
        parseable = "<container>" + string + "</container>"
        xml = ElementTree.fromstring(parseable.encode('utf-8'))

        element_to_fill.text = xml.text
        for xml_el in list(xml):
            element_to_fill.append(xml_el)
        element_to_fill.tail = xml.tail

    def flush(self):
        xliff = ElementTree.fromstring('\n'.join(self._content))
        jobs = []

        for tu in xliff.findall('.//{' + self.DEFAULT_NAMESPACE + '}trans-unit'):
            job = self._pool.apply_async(self._translate_transunit, (tu, None))
            jobs.append(job)

        for job in jobs:
            _ = job.get()

        print ElementTree.tostring(xliff, encoding='UTF-8', method='xml')

    def close(self):
        Translator.close(self)
        self._pool.terminate()


class InteractiveTranslator(Translator):
    def __init__(self, node, context_string=None, context_file=None, context_vector=None,
                 print_nbest=False, nbest_file=None):
        Translator.__init__(self, node, context_string, context_file, context_vector, print_nbest, nbest_file)<commit_msg>Improved XLIFF inner xml parsing<commit_after>import Queue
import sys
import threading
from xml.etree import ElementTree

import copy
import requests

from cli import IllegalArgumentException, IllegalStateException
from cli.libs import multithread
from cli.mmt.cluster import ClusterNode

__author__ = '<NAME>'


class Translator:
    def __init__(self, node, context_string=None, context_file=None, context_vector=None,
                 print_nbest=None, nbest_file=None):
        self.source_lang = node.engine.source_lang
        self.target_lang = node.engine.target_lang
        self._priority = ClusterNode.Api.PRIORITY_BACKGROUND
        self._api = node.api
        self._print_nbest = print_nbest

        self._nbest_out = None
        self._nbest_file = None

        if print_nbest:
            self._nbest_out = sys.stdout

            if nbest_file is not None:
                self._nbest_out = self._nbest_file = open(nbest_file, 'wb')

        self._context = None

        if context_string is not None:
            self._context = self._api.get_context_s(self.source_lang, self.target_lang, context_string)
        elif context_file is not None:
            self._context = self._api.get_context_f(self.source_lang, self.target_lang, context_file)
        elif context_vector is not None:
            self._context = self.__parse_context_vector(context_vector)

    @staticmethod
    def __parse_context_vector(text):
        context = []

        try:
            for score in text.split(','):
                id, value = score.split(':', 2)
                value = float(value)

                context.append({
                    'memory': int(id),
                    'score': value
                })
        except ValueError:
            raise IllegalArgumentException('invalid context weights map: ' + text)

        return context

    @staticmethod
    def _encode_translation(translation):
        return translation['translation'].encode('utf-8')

    @staticmethod
    def _encode_nbest(nbest):
        scores = []

        for name, values in nbest['scores'].iteritems():
            scores.append(name + '= ' + ' '.join([str(f) for f in values]))

        return [
            nbest['translation'],
            ' '.join(scores),
            str(nbest['totalScore'])
        ]

    def _translate(self, line, _=None):
        return self._api.translate(self.source_lang, self.target_lang, line, context=self._context,
                                   nbest=self._print_nbest, priority=self._priority)

    def execute(self, line):
        pass

    def flush(self):
        pass

    def close(self):
        if self._nbest_file is not None:
            self._nbest_file.close()


class BatchTranslator(Translator):
    def __init__(self, node, context_string=None, context_file=None, context_vector=None,
                 print_nbest=False, nbest_file=None, pool_size=100):
        Translator.__init__(self, node, context_string, context_file, context_vector, print_nbest, nbest_file)
        self._pool = multithread.Pool(pool_size)
        self._jobs = Queue.Queue(pool_size)
        self._line_id = 0
        self._printer_thread = threading.Thread(target=self._threaded_print)

        self._printer_thread.start()

    def execute(self, line):
        result = self._pool.apply_async(self._translate, (line, None))
        self._jobs.put(result, block=True)

    def flush(self):
        self._jobs.put(None, block=True)

    def _threaded_print(self):
        while True:
            job = self._jobs.get(block=True)

            if job is None:
                break

            translation = job.get()

            print self._encode_translation(translation)
            if self._print_nbest is not None:
                for nbest in translation['nbest']:
                    parts = [str(self._line_id)] + self._encode_nbest(nbest)
                    self._nbest_out.write((u' ||| '.join(parts)).encode('utf-8'))
                    self._nbest_out.write('\n')

            self._line_id += 1

    def close(self):
        self._printer_thread.join()
        Translator.close(self)
        self._pool.terminate()


class XLIFFTranslator(Translator):
    NAMESPACES = {
        'xlf': 'urn:oasis:names:tc:xliff:document:1.2',
        'sdl': 'http://sdl.com/FileTypes/SdlXliff/1.0',
        'mq': 'MQXliff'
    }
    DEFAULT_NAMESPACE = 'urn:oasis:names:tc:xliff:document:1.2'
    SDL_NAMESPACE = 'http://sdl.com/FileTypes/SdlXliff/1.0'

    def __init__(self, node, context_string=None, context_file=None, context_vector=None):
        Translator.__init__(self, node, context_string, context_file, context_vector)
        self._content = []
        self._pool = multithread.Pool(100)

        for namespace, uri in self.NAMESPACES.iteritems():
            if namespace == 'xlf':
                namespace = ''
            ElementTree.register_namespace(namespace, uri)

    def execute(self, line):
        self._content.append(line)

    def _translate_transunit(self, tu, _=None):
        # if there is a 100% sdl match
        if self._skip_translation_unit(tu):
            return None

        ns = self.NAMESPACES

        source_tag = tu.find('xlf:seg-source', ns)
        if source_tag is None:
            source_tag = tu.find('xlf:source', ns)
        target_tag = tu.find('xlf:target', ns)

        source_content, placeholders = self._get_source_content(source_tag)

        if source_content is None:
            return None

        # translation = self._translate(source_content)
        translation = {'translation': source_content}
        self._append_translation(translation['translation'], target_tag, placeholders)

        return None

    def _skip_translation_unit(self, tu):
        match = tu.find('.//sdl:seg[@percent="100"]', self.NAMESPACES)
        return True if match is not None else False

    @staticmethod
    def _get_source_content(element):
        if element is None:
            return None, None

        def _navigate(el, placeholders):
            for child in list(el):
                name = child.tag if '}' not in child.tag else child.tag.split('}', 1)[1]
                if name in ['ph', 'bpt', 'ept', 'it']:
                    clone = copy.deepcopy(child)
                    clone.tail = None
                    placeholders.append(clone)

                    child.text = None
                    child.attrib = {'id': str(len(placeholders))}
                else:
                    _navigate(child, placeholders)

            return el, placeholders

        content, _placeholders = _navigate(copy.deepcopy(element), [])
        content = ElementTree.tostring(content, encoding='utf-8', method='xml')
        content = content[content.find('>') + 1:]
        content = content[:content.rfind('</source>')]
        return (content, _placeholders) if len(content) > 0 else (None, None)

    def _append_translation(self, content, element, placeholders):
        content = '<content xmlns="%s">%s</content>' % (self.NAMESPACES['xlf'], content)
        content = ElementTree.fromstring(content.encode('utf-8'))

        # Replace placeholders
        parent_map = dict((c, p) for p in content.getiterator() for c in p)

        for i, source in enumerate(placeholders):
            target = content.find('.//%s[@id="%d"]' % (source.tag, i + 1))
            if target is not None:
                source.tail = target.tail

                parent = parent_map[target]
                parent_index = list(parent).index(target)
                parent.remove(target)
                parent.insert(parent_index, source)

        # Clear target element
        for child in list(element):
            element.remove(child)

        # Replace target content
        element.text = content.text
        element.tail = content.tail
        for child in list(content):
            content.remove(child)
            element.append(child)

    def flush(self):
        xliff = ElementTree.fromstring('\n'.join(self._content))
        jobs = []

        for tu in xliff.findall('.//xlf:trans-unit', self.NAMESPACES):
            job = self._pool.apply_async(self._translate_transunit, (tu, None))
            jobs.append(job)

        for job in jobs:
            _ = job.get()

        print ElementTree.tostring(xliff, encoding='UTF-8', method='xml')

    def close(self):
        Translator.close(self)
        self._pool.terminate()


class InteractiveTranslator(Translator):
    def __init__(self, node, context_string=None, context_file=None, context_vector=None,
                 print_nbest=False, nbest_file=None):
        Translator.__init__(self, node, context_string, context_file, context_vector, print_nbest, nbest_file)
{'fraction_non_alphanumeric': 0.10397536394176932, 'fraction_numerical': 0.0030235162374020155, 'mean_word_length': 2.3279299422396122, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 17}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2711836', 'n_tokens_mistral': 5319, 'n_tokens_neox': 5083, 'n_words': 1190}