// Path: src/main/java/io/aif/language/common/IExtractor.java
// @FunctionalInterface
// public interface IExtractor<T1, T2> extends Function<T1, Optional<T2>> {
// 
//   public Optional<T2> extract(final T1 from);
// 
//   @Override
//   public default Optional<T2> apply(final T1 t1) {
//     return extract(t1);
//   }
// 
// }
// 
// Path: src/main/java/io/aif/language/common/settings/ISettings.java
// public interface ISettings {
// 
//   @Deprecated
//   public ISettings SETTINGS = Guice.createInjector(new SettingsModule()).getInstance(ISettings.class);
// 
//   public String getVersion();
// 
//   public int recommendedMinimumTokensInputCount();
// 
//   public boolean useIsAlphabeticMethod();
// 
//   public double thresholdPForSeparatorCharacterInSecondFilter();
// 
//   public int minimalValuableTokenSizeForSentenceSplit();
// 
//   public int minimumCharacterObservationsCountForMakingCharacterValuableDuringSentenceSplitting();
// 
//   public double thresholdPFirstFilterForSeparatorCharacter();
// 
//   public double splitterCharactersGrouperSearchStep();
// 
//   public double splitterCharactersGrouperInitSearchPValue();
// 
//   public double wordSetDictComparatorThreshold();
// 
//   public double recursiveSubstringComparatorWeight();
// 
//   public double simpleTokenComparatorWeight();
// 
//   public double characterDensityComparatorWeight();
// 
//   public String predefinedSeparators();
// }
// 
// Path: src/main/java/io/aif/language/common/settings/SettingsModule.java
// public class SettingsModule extends AbstractModule {
// 
//   @Override
//   protected void configure() {
//     bind(ISettings.class).toProvider(new PropertyBasedSettings.Provider());
//   }
// 
// }


import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Guice;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import io.aif.language.common.IExtractor;
import io.aif.language.common.VisibilityReducedForCLI;
import io.aif.language.common.settings.ISettings;
import io.aif.language.common.settings.SettingsModule;
import io.aif.language.token.TokenMappers;
import static java.lang.Character.isAlphabetic;
import static java.util.Collections.emptyList;
import static java.util.stream.Collectors.toList;
package io.aif.language.sentence.separators.extractors;





// TODO(#263): StatSeparatorExtractor should be documented.
// TODO(#265): Publish article about the algorithm of separators extractors.
class StatSeparatorExtractor implements ISeparatorExtractor {

  private static final IExtractor<String, Character> END_CHARACTER_EXTRACTOR =
    token -> Optional.of(token.charAt(token.length() - 1));

  private static final IExtractor<String, Character> CHARACTER_BEFORE_END_CHARACTER_EXTRACTOR =
    token -> Optional.of(token.charAt(token.length() - 2));

  private static final IExtractor<String, Character> START_CHARACTER_EXTRACTOR = token -> Optional.of(token.charAt(0));

  private static final IExtractor<String, Character> CHARACTER_AFTER_START_CHARACTER_EXTRACTOR =
    token -> Optional.of(token.charAt(1));

  private static final StatDataExtractor END_CHARACTER_STAT_DATA_EXTRACTOR =
    new StatDataExtractor(END_CHARACTER_EXTRACTOR, CHARACTER_BEFORE_END_CHARACTER_EXTRACTOR);

  private static final StatDataExtractor START_CHARACTER_STAT_DATA_EXTRACTOR =
    new StatDataExtractor(START_CHARACTER_EXTRACTOR, CHARACTER_AFTER_START_CHARACTER_EXTRACTOR);

// Path: src/main/java/io/aif/language/common/IExtractor.java
// @FunctionalInterface
// public interface IExtractor<T1, T2> extends Function<T1, Optional<T2>> {
// 
//   public Optional<T2> extract(final T1 from);
// 
//   @Override
//   public default Optional<T2> apply(final T1 t1) {
//     return extract(t1);
//   }
// 
// }
// 
// Path: src/main/java/io/aif/language/common/settings/ISettings.java
// public interface ISettings {
// 
//   @Deprecated
//   public ISettings SETTINGS = Guice.createInjector(new SettingsModule()).getInstance(ISettings.class);
// 
//   public String getVersion();
// 
//   public int recommendedMinimumTokensInputCount();
// 
//   public boolean useIsAlphabeticMethod();
// 
//   public double thresholdPForSeparatorCharacterInSecondFilter();
// 
//   public int minimalValuableTokenSizeForSentenceSplit();
// 
//   public int minimumCharacterObservationsCountForMakingCharacterValuableDuringSentenceSplitting();
// 
//   public double thresholdPFirstFilterForSeparatorCharacter();
// 
//   public double splitterCharactersGrouperSearchStep();
// 
//   public double splitterCharactersGrouperInitSearchPValue();
// 
//   public double wordSetDictComparatorThreshold();
// 
//   public double recursiveSubstringComparatorWeight();
// 
//   public double simpleTokenComparatorWeight();
// 
//   public double characterDensityComparatorWeight();
// 
//   public String predefinedSeparators();
// }
// 
// Path: src/main/java/io/aif/language/common/settings/SettingsModule.java
// public class SettingsModule extends AbstractModule {
// 
//   @Override
//   protected void configure() {
//     bind(ISettings.class).toProvider(new PropertyBasedSettings.Provider());
//   }
// 
// }


// Path: src/main/java/io/aif/language/sentence/separators/extractors/StatSeparatorExtractor.java
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Guice;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import io.aif.language.common.IExtractor;
import io.aif.language.common.VisibilityReducedForCLI;
import io.aif.language.common.settings.ISettings;
import io.aif.language.common.settings.SettingsModule;
import io.aif.language.token.TokenMappers;
import static java.lang.Character.isAlphabetic;
import static java.util.Collections.emptyList;
import static java.util.stream.Collectors.toList;

package io.aif.language.sentence.separators.extractors;





// TODO(#263): StatSeparatorExtractor should be documented.
// TODO(#265): Publish article about the algorithm of separators extractors.
class StatSeparatorExtractor implements ISeparatorExtractor {

  private static final IExtractor<String, Character> END_CHARACTER_EXTRACTOR =
    token -> Optional.of(token.charAt(token.length() - 1));

  private static final IExtractor<String, Character> CHARACTER_BEFORE_END_CHARACTER_EXTRACTOR =
    token -> Optional.of(token.charAt(token.length() - 2));

  private static final IExtractor<String, Character> START_CHARACTER_EXTRACTOR = token -> Optional.of(token.charAt(0));

  private static final IExtractor<String, Character> CHARACTER_AFTER_START_CHARACTER_EXTRACTOR =
    token -> Optional.of(token.charAt(1));

  private static final StatDataExtractor END_CHARACTER_STAT_DATA_EXTRACTOR =
    new StatDataExtractor(END_CHARACTER_EXTRACTOR, CHARACTER_BEFORE_END_CHARACTER_EXTRACTOR);

  private static final StatDataExtractor START_CHARACTER_STAT_DATA_EXTRACTOR =
    new StatDataExtractor(START_CHARACTER_EXTRACTOR, CHARACTER_AFTER_START_CHARACTER_EXTRACTOR);

  private static final ISettings SETTINGS = Guice.createInjector(new SettingsModule()).getInstance(ISettings.class);