<commit_before>package com.aif.language.sentence;

import com.aif.language.common.ISplitter;

import java.util.*;
import java.util.stream.Collectors;

public class SentenceSplitter implements ISplitter<List<String>, List<String>> {

    private         final   ISentenceSeparatorExtractor sentenceSeparatorExtractor;

    public SentenceSplitter(final ISentenceSeparatorExtractor sentenceSeparatorExtractor) {
        this.sentenceSeparatorExtractor = sentenceSeparatorExtractor;
    }

    public SentenceSplitter() {
        this(ISentenceSeparatorExtractor.Type.PREDEFINED.getInstance());
    }

    @Override
    public List<List<String>> split(final List<String> tokens) {
        final Optional<List<Character>> optionalSeparators = sentenceSeparatorExtractor.extract(tokens);

        if (!optionalSeparators.isPresent()) {
            return new ArrayList<List<String>>(){{add(tokens);}};
        }

        final List<Character> separators = optionalSeparators.get();

        final List<Boolean> listOfPositions = mapToBooleans(tokens, separators);

        final SentenceIterator sentenceIterator = new SentenceIterator(tokens, listOfPositions);

        final List<List<String>> sentences = new ArrayList<>();
        while (sentenceIterator.hasNext()) {
            sentences.add(sentenceIterator.next());
        }

        sentences.forEach(sentence -> prepareSentences(sentence, separators));

        return sentences
                .parallelStream()
                .map(sentence -> SentenceSplitter.prepareSentences(sentence, separators))
                .collect(Collectors.toList());
    }

    private static List<String> prepareSentences(final List<String> sentence, final List<Character> separators) {
        final String lastToken = sentence.get(sentence.size() - 1);
        int index = lastToken.length() - 1;
        while (index > 0 && separators.contains(lastToken.charAt(index))) {
            index--;
        }
        index++;
        if (index < lastToken.length()) {
            final String leftToken = lastToken.substring(0, index);
            final String rightToken = lastToken.substring(index);
            final List<String> preparedSentance = new ArrayList<>(sentence.size() + 1);
            for (int i = 0; i < sentence.size() - 1; i++) {
                preparedSentance.add(sentence.get(i));
            }
            preparedSentance.add(leftToken);
            preparedSentance.add(rightToken);
            return preparedSentance;
        }
        return sentence;
    }

    private List<Boolean> mapToBooleans(final List<String> tokens, final List<Character> separators) {
        return tokens.stream()
                .map(token -> separators.contains(token.charAt(token.length() - 1)))
                .collect(Collectors.toList());
    }

    private static class SentenceIterator implements Iterator<List<String>> {

        private final   List<String>    tokens;

        private final   List<Boolean>   endTokens;

        private         int             currentPosition = 0;

        private SentenceIterator(List<String> tokens, List<Boolean> endTokens) {
            this.tokens = tokens;<commit_msg>fixing sentence splitters extractor<commit_after>package com.aif.language.sentence;

import com.aif.language.common.ISplitter;

import java.util.*;
import java.util.stream.Collectors;

public class SentenceSplitter implements ISplitter<List<String>, List<String>> {

    private         final   ISentenceSeparatorExtractor sentenceSeparatorExtractor;

    public SentenceSplitter(final ISentenceSeparatorExtractor sentenceSeparatorExtractor) {
        this.sentenceSeparatorExtractor = sentenceSeparatorExtractor;
    }

    public SentenceSplitter() {
        this(ISentenceSeparatorExtractor.Type.PREDEFINED.getInstance());
    }

    @Override
    public List<List<String>> split(final List<String> tokens) {
        final Optional<List<Character>> optionalSeparators = sentenceSeparatorExtractor.extract(tokens);

        if (!optionalSeparators.isPresent()) {
            return new ArrayList<List<String>>(){{add(tokens);}};
        }

        final List<Character> separators = optionalSeparators.get();

        final List<Boolean> listOfPositions = mapToBooleans(tokens, separators);

        final SentenceIterator sentenceIterator = new SentenceIterator(tokens, listOfPositions);

        final List<List<String>> sentences = new ArrayList<>();
        while (sentenceIterator.hasNext()) {
            sentences.add(sentenceIterator.next());
        }

        sentences.forEach(sentence -> prepareSentences(sentence, separators));

        return sentences
                .parallelStream()
                .map(sentence -> SentenceSplitter.prepareSentences(sentence, separators))
                .collect(Collectors.toList());
    }

    private static List<String> prepareSentences(final List<String> sentence, final List<Character> separators) {
        final List<String> preparedTokens = new ArrayList<>();

        for (String token: sentence) {
            preparedTokens.addAll(prepareToken(token, separators));
        }

        return preparedTokens;
    }

    private static List<String> prepareToken(final String token, final List<Character> separators) {
        final List<String> tokens = new ArrayList<>(3);
        boolean prevCharacterIsSeparator = separators.contains(token.charAt(0));
        int lastIndex = 0;
        for (int i = 1; i < token.length(); i++) {
            final boolean cuurentCharacterSeparator = separators.contains(token.charAt(i));
            if (prevCharacterIsSeparator != cuurentCharacterSeparator) {
                tokens.add(token.substring(lastIndex, i));
                lastIndex = i;
                prevCharacterIsSeparator = cuurentCharacterSeparator;
            }
        }
        tokens.add(token.substring(lastIndex));
        return tokens;
    }

    private List<Boolean> mapToBooleans(final List<String> tokens, final List<Character> separators) {
        return tokens.stream()
                .map(token -> separators.contains(token.charAt(token.length() - 1)))
                .collect(Collectors.toList());
    }

    private static class SentenceIterator implements Iterator<List<String>> {

        private final   List<String>    tokens;

        private final   List<Boolean>   endTokens;

        private         int             currentPosition = 0;

        private SentenceIterator(List<String> tokens, List<Boolean> endTokens) {
            this.tokens = tokens;
{'fraction_non_alphanumeric': 0.1004601226993865, 'fraction_numerical': 0.0023006134969325155, 'mean_word_length': 2.8723277909738716, 'pattern_counts': {'":': 0, '<': 68, '<?xml version=': 0, '>': 72, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4668598', 'n_tokens_mistral': 1718, 'n_tokens_neox': 1650, 'n_words': 352}