<commit_before>package eu.modernmt.processing.splitter;

import eu.modernmt.lang.Language;
import eu.modernmt.model.Sentence;
import eu.modernmt.model.Word;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * A SentenceSplitter is a preprocessing component that is capable of
 * - receiving a list of tokens obtained from a text
 * - decide whether they form multiple sentences
 * - return the positions after which a sentence ends (splits)
 */
public abstract class SentenceSplitter {

    private static final SentenceSplitter DEFAULT_IMPL = new DefaultSentenceSplitter();
    private static final Map<String, SentenceSplitter> IMPLEMENTATIONS = new HashMap<>();
    private static final int MIN_SENTENCE_SIZE = 2000;

    public static SentenceSplitter forLanguage(Language language) {
        return IMPLEMENTATIONS.getOrDefault(language.getLanguage(), DEFAULT_IMPL);
    }

    /**
     * This method splits a sentence into multiple sentences by looking for splits among its words.
     * A split is a Word in correspondence of which a sentence ends.
     * So, if splits are found, it means that multiple sentences are contained in the passed text,
     * (and they should be handled separately by the translation engine.)
     *
     * @param sentence the text to find the splits of, in the form of a Word array
     * @return a List containing the split positions in the array.
     */
    public Sentence[] split(Sentence sentence) {
        Word[] originalWords = sentence.getWords();
        List<Sentence> splitSentences = new ArrayList<>();

        int prevSplit = -1;
        for (int i = 0; i < originalWords.length; i++) {
            /* i - prevSplit is the amount of words seen since the last split. Do not even check the split if it is too low */
            if (i - prevSplit < MIN_SENTENCE_SIZE)
                continue;

            if (isSplit(sentence, i)) {
                Word[] newSplitSentenceWords = new Word[i - prevSplit];
                for (int j = 0; j < i - prevSplit; j++)
                    newSplitSentenceWords[j] = originalWords[j + prevSplit + 1];
                splitSentences.add(new Sentence(newSplitSentenceWords));
                prevSplit = i;
            }
        }

        //handle the last sentence
        Word[] lastSplitSentenceWords = new Word[originalWords.length - 1 - prevSplit];
        for (int j = 0; j < originalWords.length - 1 - prevSplit; j++)
            lastSplitSentenceWords[j] = originalWords[j + prevSplit + 1];
        splitSentences.add(new Sentence(lastSplitSentenceWords));

        Sentence[] array = new Sentence[splitSentences.size()];
        return splitSentences.toArray(array);
    }

    /**
     * This method checks if the a Word at a certain index in a Sentence is a split.
     *
     * @param sentence  the sentence to check the presence of splits in
     * @param wordIndex the index of the word to check
     * @return TRUE is the current word is a split, FALSE otherwise.
     */
    protected abstract boolean isSplit(Sentence sentence, int wordIndex);
}
<commit_msg>Restored acceptable value for SentenceSplitter.MIN_SENTENCE_SIZE<commit_after>package eu.modernmt.processing.splitter;

import eu.modernmt.lang.Language;
import eu.modernmt.model.Sentence;
import eu.modernmt.model.Word;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
 * A SentenceSplitter is a preprocessing component that is capable of
 * - receiving a list of tokens obtained from a text
 * - decide whether they form multiple sentences
 * - return the positions after which a sentence ends (splits)
 */
public abstract class SentenceSplitter {

    private static final SentenceSplitter DEFAULT_IMPL = new DefaultSentenceSplitter();
    private static final Map<String, SentenceSplitter> IMPLEMENTATIONS = new HashMap<>();
    private static final int MIN_SENTENCE_SIZE = 128;

    public static SentenceSplitter forLanguage(Language language) {
        return IMPLEMENTATIONS.getOrDefault(language.getLanguage(), DEFAULT_IMPL);
    }

    /**
     * This method splits a sentence into multiple sentences by looking for splits among its words.
     * A split is a Word in correspondence of which a sentence ends.
     * So, if splits are found, it means that multiple sentences are contained in the passed text,
     * (and they should be handled separately by the translation engine.)
     *
     * @param sentence the text to find the splits of, in the form of a Word array
     * @return a List containing the split positions in the array.
     */
    public Sentence[] split(Sentence sentence) {
        Word[] originalWords = sentence.getWords();
        List<Sentence> splitSentences = new ArrayList<>();

        int prevSplit = -1;
        for (int i = 0; i < originalWords.length; i++) {
            /* i - prevSplit is the amount of words seen since the last split. Do not even check the split if it is too low */
            if (i - prevSplit < MIN_SENTENCE_SIZE)
                continue;

            if (isSplit(sentence, i)) {
                Word[] newSplitSentenceWords = new Word[i - prevSplit];
                for (int j = 0; j < i - prevSplit; j++)
                    newSplitSentenceWords[j] = originalWords[j + prevSplit + 1];
                splitSentences.add(new Sentence(newSplitSentenceWords));
                prevSplit = i;
            }
        }

        //handle the last sentence
        Word[] lastSplitSentenceWords = new Word[originalWords.length - 1 - prevSplit];
        for (int j = 0; j < originalWords.length - 1 - prevSplit; j++)
            lastSplitSentenceWords[j] = originalWords[j + prevSplit + 1];
        splitSentences.add(new Sentence(lastSplitSentenceWords));

        Sentence[] array = new Sentence[splitSentences.size()];
        return splitSentences.toArray(array);
    }

    /**
     * This method checks if the a Word at a certain index in a Sentence is a split.
     *
     * @param sentence  the sentence to check the presence of splits in
     * @param wordIndex the index of the word to check
     * @return TRUE is the current word is a split, FALSE otherwise.
     */
    protected abstract boolean isSplit(Sentence sentence, int wordIndex);
}

{'fraction_non_alphanumeric': 0.08020965692503176, 'fraction_numerical': 0.003653113087674714, 'mean_word_length': 2.7751798561151078, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1120559', 'n_tokens_mistral': 1753, 'n_tokens_neox': 1636, 'n_words': 646}