<commit_before>package eu.modernmt.facade;

import com.hazelcast.core.HazelcastException;
import eu.modernmt.cluster.ClusterNode;
import eu.modernmt.cluster.TranslationTask;
import eu.modernmt.cluster.error.SystemShutdownException;
import eu.modernmt.context.ContextAnalyzer;
import eu.modernmt.context.ContextAnalyzerException;
import eu.modernmt.decoder.Decoder;
import eu.modernmt.decoder.DecoderException;
import eu.modernmt.decoder.DecoderWithNBest;
import eu.modernmt.decoder.TranslationTimeoutException;
import eu.modernmt.engine.Engine;
import eu.modernmt.lang.Language;
import eu.modernmt.lang.LanguageDirection;
import eu.modernmt.lang.LanguageIndex;
import eu.modernmt.lang.UnsupportedLanguageException;
import eu.modernmt.model.ContextVector;
import eu.modernmt.model.Priority;
import eu.modernmt.model.Sentence;
import eu.modernmt.model.Translation;
import eu.modernmt.model.corpus.Corpus;
import eu.modernmt.model.corpus.impl.StringCorpus;
import eu.modernmt.model.corpus.impl.parallel.FileCorpus;
import eu.modernmt.processing.Postprocessor;
import eu.modernmt.processing.Preprocessor;
import eu.modernmt.processing.ProcessingException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.File;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;

/**
 * Created by davide on 31/01/17.
 */
public class TranslationFacade {

    private static final Logger logger = LogManager.getLogger(TranslationFacade.class);

    // =============================
    //  Translation
    // =============================

    public Translation get(UUID user, LanguageDirection direction, String text, Priority priority, long timeout) throws ProcessingException, DecoderException {
        return get(user, direction, text, null, 0, priority, timeout);
    }

    public Translation get(UUID user, LanguageDirection direction, String text, ContextVector translationContext, Priority priority, long timeout) throws ProcessingException, DecoderException {
        return get(user, direction, text, translationContext, 0, priority, timeout);
    }

    public Translation get(UUID user, LanguageDirection direction, String text, int nbest, Priority priority, long timeout) throws ProcessingException, DecoderException {
        return get(user, direction, text, null, nbest, priority, timeout);
    }

    public Translation get(UUID user, LanguageDirection direction, String text, ContextVector translationContext, int nbest, Priority priority, long timeout) throws ProcessingException, DecoderException {
        direction = mapLanguagePair(direction);
        if (nbest > 0)
            ensureDecoderSupportsNBest();

        Engine engine = ModernMT.getNode().getEngine();
        Preprocessor preprocessor = engine.getPreprocessor();
        Postprocessor postprocessor = engine.getPostprocessor();

        // Pre-processing text
        Sentence sentence = preprocessor.process(direction, text);

        // Translating
        Translation translation;
        long expirationTimestamp = timeout > 0 ? (System.currentTimeMillis() + timeout) : 0L;

        try {
            translation = insecureGet(user, direction, sentence, translationContext, nbest, priority, expirationTimestamp);
        } catch (DecoderException | HazelcastException e) {
            logger.warn("Translation failed, retry after delay", e);

            try {
                Thread.sleep(50);
            } catch (InterruptedException e1) {
                // Ignore it
            }

            translation = insecureGet(user, direction, sentence, translationContext, nbest, priority, expirationTimestamp);
        }

        // Post-processing translation
        postprocessor.process(direction, translation);

        if (translation.hasNbest()) {
            List<Translation> hypotheses = translation.getNbest();<commit_msg>Prevent translation retry if Timeout occurred<commit_after>package eu.modernmt.facade;

import com.hazelcast.core.HazelcastException;
import eu.modernmt.cluster.ClusterNode;
import eu.modernmt.cluster.TranslationTask;
import eu.modernmt.cluster.error.SystemShutdownException;
import eu.modernmt.context.ContextAnalyzer;
import eu.modernmt.context.ContextAnalyzerException;
import eu.modernmt.decoder.Decoder;
import eu.modernmt.decoder.DecoderException;
import eu.modernmt.decoder.DecoderWithNBest;
import eu.modernmt.decoder.TranslationTimeoutException;
import eu.modernmt.engine.Engine;
import eu.modernmt.lang.Language;
import eu.modernmt.lang.LanguageDirection;
import eu.modernmt.lang.LanguageIndex;
import eu.modernmt.lang.UnsupportedLanguageException;
import eu.modernmt.model.ContextVector;
import eu.modernmt.model.Priority;
import eu.modernmt.model.Sentence;
import eu.modernmt.model.Translation;
import eu.modernmt.model.corpus.Corpus;
import eu.modernmt.model.corpus.impl.StringCorpus;
import eu.modernmt.model.corpus.impl.parallel.FileCorpus;
import eu.modernmt.processing.Postprocessor;
import eu.modernmt.processing.Preprocessor;
import eu.modernmt.processing.ProcessingException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;

import java.io.File;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import java.util.concurrent.RejectedExecutionException;

/**
 * Created by davide on 31/01/17.
 */
public class TranslationFacade {

    private static final Logger logger = LogManager.getLogger(TranslationFacade.class);

    // =============================
    //  Translation
    // =============================

    public Translation get(UUID user, LanguageDirection direction, String text, Priority priority, long timeout) throws ProcessingException, DecoderException {
        return get(user, direction, text, null, 0, priority, timeout);
    }

    public Translation get(UUID user, LanguageDirection direction, String text, ContextVector translationContext, Priority priority, long timeout) throws ProcessingException, DecoderException {
        return get(user, direction, text, translationContext, 0, priority, timeout);
    }

    public Translation get(UUID user, LanguageDirection direction, String text, int nbest, Priority priority, long timeout) throws ProcessingException, DecoderException {
        return get(user, direction, text, null, nbest, priority, timeout);
    }

    public Translation get(UUID user, LanguageDirection direction, String text, ContextVector translationContext, int nbest, Priority priority, long timeout) throws ProcessingException, DecoderException {
        direction = mapLanguagePair(direction);
        if (nbest > 0)
            ensureDecoderSupportsNBest();

        Engine engine = ModernMT.getNode().getEngine();
        Preprocessor preprocessor = engine.getPreprocessor();
        Postprocessor postprocessor = engine.getPostprocessor();

        // Pre-processing text
        Sentence sentence = preprocessor.process(direction, text);

        // Translating
        Translation translation;
        long expirationTimestamp = timeout > 0 ? (System.currentTimeMillis() + timeout) : 0L;

        try {
            translation = insecureGet(user, direction, sentence, translationContext, nbest, priority, expirationTimestamp);
        } catch (DecoderException | HazelcastException e) {
            if (e instanceof TranslationTimeoutException)
                throw e;

            logger.warn("Translation failed, retry after delay", e);

            try {
                Thread.sleep(50);
            } catch (InterruptedException e1) {
                // Ignore it
            }

            translation = insecureGet(user, direction, sentence, translationContext, nbest, priority, expirationTimestamp);
        }

        // Post-processing translation
        postprocessor.process(direction, translation);

        if (translation.hasNbest()) {
            List<Translation> hypotheses = translation.getNbest();
{'fraction_non_alphanumeric': 0.10161014881678458, 'fraction_numerical': 0.003903391070992925, 'mean_word_length': 4.0830750154990705, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3227448', 'n_tokens_mistral': 2190, 'n_tokens_neox': 2048, 'n_words': 516}