<commit_before>/* LanguageTool, a natural language style checker 
 * Copyright (C) 2012 <NAME> (http://www.languagetool.org)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.rules.spelling;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.jetbrains.annotations.Nullable;
import org.languagetool.*;
import org.languagetool.languagemodel.BaseLanguageModel;
import org.languagetool.languagemodel.LanguageModel;
import org.languagetool.rules.ITSIssueType;
import org.languagetool.rules.Rule;
import org.languagetool.rules.RuleMatch;
import org.languagetool.rules.SuggestedReplacement;
import org.languagetool.rules.patterns.PatternToken;
import org.languagetool.rules.patterns.PatternTokenBuilder;
import org.languagetool.rules.spelling.suggestions.SuggestionsOrderer;
import org.languagetool.rules.spelling.suggestions.SuggestionsOrdererFeatureExtractor;
import org.languagetool.rules.spelling.suggestions.SuggestionsRanker;
import org.languagetool.tagging.disambiguation.rules.DisambiguationPatternRule;
import org.languagetool.tokenizers.WordTokenizer;
import org.languagetool.tools.StringTools;

import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

/**
 * An abstract rule for spellchecking rules.
 *
 * @author <NAME>
 */
public abstract class SpellingCheckRule extends Rule {

  /**
   * The string {@code LanguageTool}.
   * @since 2.3
   */
  public static final String LANGUAGETOOL = "LanguageTool";
  /**
   * The string {@code LanguageTooler}.
   * @since 4.4
   */
  public static final String LANGUAGETOOLER = "LanguageTooler";

  protected final Language language;

  /**
   * @since 4.5
   * For rules from @see Language.getRelevantLanguageModelCapableRules
   * Optional, allows e.g. better suggestions when set
   */
  @Nullable
  @Experimental
  protected LanguageModel languageModel;
  protected final CachingWordListLoader wordListLoader = new CachingWordListLoader();

  private static final String SPELLING_IGNORE_FILE = "/hunspell/ignore.txt";
  private static final String SPELLING_FILE = "/hunspell/spelling.txt";
  private static final String SPELLING_PROHIBIT_FILE = "/hunspell/prohibit.txt";
  private static final String SPELLING_FILE_VARIANT = null;
  private static final Comparator<String> STRING_LENGTH_COMPARATOR = Comparator.comparingInt(String::length);

  private final UserConfig userConfig;
  private final Set<String> wordsToBeIgnored = new HashSet<>();
  private final Set<String> wordsToBeProhibited = new HashSet<>();
  private final List<RuleWithLanguage> altRules;

  private Map<String,Set<String>> wordsToBeIgnoredDictionary = new HashMap<>();
  private Map<String,Set<String>> wordsToBeIgnoredDictionaryIgnoreCase = new HashMap<>();
  
  private List<DisambiguationPatternRule> antiPatterns = new ArrayList<>();
  private boolean considerIgnoreWords = true;
  private boolean convertsCase = false;
  protected int ignoreWordsWithLength = 0;

  public SpellingCheckRule(ResourceBundle messages, Language language, UserConfig userConfig) {
    this(messages, language, userConfig, Collections.emptyList());
  }

  /**
   * @since 4.4
   */
  public SpellingCheckRule(ResourceBundle messages, Language language, UserConfig userConfig, List<Language> altLanguages) {
    this(messages, language, userConfig, altLanguages, null);
  }

  /**
   * @since 4.5
   */
  @Experimental
  public SpellingCheckRule(ResourceBundle messages, Language language, UserConfig userConfig, List<Language> altLanguages, @Nullable LanguageModel languageModel) {
    super(messages);
    this.language = language;
    this.userConfig = userConfig;
    this.languageModel = languageModel;
    if (userConfig != null) {
      wordsToBeIgnored.addAll(userConfig.getAcceptedWords());
    }
    this.altRules = getAlternativeLangSpellingRules(altLanguages);
    setLocQualityIssueType(ITSIssueType.Misspelling);
  }

  /**
   *
   * @param word misspelled word that suggestions should be generated for
   * @param userCandidates candidates from personal dictionary
   * @param candidates candidates from default dictionary
   * @param orderer model to rank suggestions / extract features, or null
   * @param match rule match to add suggestions to
   */
  protected static void addSuggestionsToRuleMatch(String word, List<String> userCandidates, List<String> candidates,
                                                  @Nullable SuggestionsOrderer orderer, RuleMatch match) {
    AnalyzedSentence sentence = match.getSentence();
    int startPos = match.getFromPos();
    long startTime = System.currentTimeMillis();
    if (orderer != null && orderer.isMlAvailable()) {
      if (orderer instanceof SuggestionsRanker) {
        // don't rank words form user dictionary, assign confidence 0.0, but add at start
        // hard to ensure performance on unknown words
        SuggestionsRanker ranker = (SuggestionsRanker) orderer;
        List<SuggestedReplacement> defaultSuggestions = ranker.orderSuggestions(
          candidates, word, sentence, startPos);
        if (defaultSuggestions.isEmpty()) {
          // could not rank for some reason
        } else {
          if (userCandidates.isEmpty()) {
            match.setAutoCorrect(ranker.shouldAutoCorrect(defaultSuggestions));
            match.setSuggestedReplacementObjects(defaultSuggestions);
          } else {
            List<SuggestedReplacement> combinedSuggestions = new ArrayList<>();
            for (String wordFromUserDict : userCandidates) {
              SuggestedReplacement s = new SuggestedReplacement(wordFromUserDict);<commit_msg>comment unused line<commit_after>/* LanguageTool, a natural language style checker 
 * Copyright (C) 2012 <NAME> (http://www.languagetool.org)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.rules.spelling;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.Pair;
import org.jetbrains.annotations.Nullable;
import org.languagetool.*;
import org.languagetool.languagemodel.BaseLanguageModel;
import org.languagetool.languagemodel.LanguageModel;
import org.languagetool.rules.ITSIssueType;
import org.languagetool.rules.Rule;
import org.languagetool.rules.RuleMatch;
import org.languagetool.rules.SuggestedReplacement;
import org.languagetool.rules.patterns.PatternToken;
import org.languagetool.rules.patterns.PatternTokenBuilder;
import org.languagetool.rules.spelling.suggestions.SuggestionsOrderer;
import org.languagetool.rules.spelling.suggestions.SuggestionsOrdererFeatureExtractor;
import org.languagetool.rules.spelling.suggestions.SuggestionsRanker;
import org.languagetool.tagging.disambiguation.rules.DisambiguationPatternRule;
import org.languagetool.tokenizers.WordTokenizer;
import org.languagetool.tools.StringTools;

import java.io.IOException;
import java.util.*;
import java.util.stream.Collectors;

/**
 * An abstract rule for spellchecking rules.
 *
 * @author <NAME>
 */
public abstract class SpellingCheckRule extends Rule {

  /**
   * The string {@code LanguageTool}.
   * @since 2.3
   */
  public static final String LANGUAGETOOL = "LanguageTool";
  /**
   * The string {@code LanguageTooler}.
   * @since 4.4
   */
  public static final String LANGUAGETOOLER = "LanguageTooler";

  protected final Language language;

  /**
   * @since 4.5
   * For rules from @see Language.getRelevantLanguageModelCapableRules
   * Optional, allows e.g. better suggestions when set
   */
  @Nullable
  @Experimental
  protected LanguageModel languageModel;
  protected final CachingWordListLoader wordListLoader = new CachingWordListLoader();

  private static final String SPELLING_IGNORE_FILE = "/hunspell/ignore.txt";
  private static final String SPELLING_FILE = "/hunspell/spelling.txt";
  private static final String SPELLING_PROHIBIT_FILE = "/hunspell/prohibit.txt";
  private static final String SPELLING_FILE_VARIANT = null;
  private static final Comparator<String> STRING_LENGTH_COMPARATOR = Comparator.comparingInt(String::length);

  private final UserConfig userConfig;
  private final Set<String> wordsToBeIgnored = new HashSet<>();
  private final Set<String> wordsToBeProhibited = new HashSet<>();
  private final List<RuleWithLanguage> altRules;

  private Map<String,Set<String>> wordsToBeIgnoredDictionary = new HashMap<>();
  private Map<String,Set<String>> wordsToBeIgnoredDictionaryIgnoreCase = new HashMap<>();
  
  private List<DisambiguationPatternRule> antiPatterns = new ArrayList<>();
  private boolean considerIgnoreWords = true;
  private boolean convertsCase = false;
  protected int ignoreWordsWithLength = 0;

  public SpellingCheckRule(ResourceBundle messages, Language language, UserConfig userConfig) {
    this(messages, language, userConfig, Collections.emptyList());
  }

  /**
   * @since 4.4
   */
  public SpellingCheckRule(ResourceBundle messages, Language language, UserConfig userConfig, List<Language> altLanguages) {
    this(messages, language, userConfig, altLanguages, null);
  }

  /**
   * @since 4.5
   */
  @Experimental
  public SpellingCheckRule(ResourceBundle messages, Language language, UserConfig userConfig, List<Language> altLanguages, @Nullable LanguageModel languageModel) {
    super(messages);
    this.language = language;
    this.userConfig = userConfig;
    this.languageModel = languageModel;
    if (userConfig != null) {
      wordsToBeIgnored.addAll(userConfig.getAcceptedWords());
    }
    this.altRules = getAlternativeLangSpellingRules(altLanguages);
    setLocQualityIssueType(ITSIssueType.Misspelling);
  }

  /**
   *
   * @param word misspelled word that suggestions should be generated for
   * @param userCandidates candidates from personal dictionary
   * @param candidates candidates from default dictionary
   * @param orderer model to rank suggestions / extract features, or null
   * @param match rule match to add suggestions to
   */
  protected static void addSuggestionsToRuleMatch(String word, List<String> userCandidates, List<String> candidates,
                                                  @Nullable SuggestionsOrderer orderer, RuleMatch match) {
    AnalyzedSentence sentence = match.getSentence();
    int startPos = match.getFromPos();
    //long startTime = System.currentTimeMillis();
    if (orderer != null && orderer.isMlAvailable()) {
      if (orderer instanceof SuggestionsRanker) {
        // don't rank words form user dictionary, assign confidence 0.0, but add at start
        // hard to ensure performance on unknown words
        SuggestionsRanker ranker = (SuggestionsRanker) orderer;
        List<SuggestedReplacement> defaultSuggestions = ranker.orderSuggestions(
          candidates, word, sentence, startPos);
        if (defaultSuggestions.isEmpty()) {
          // could not rank for some reason
        } else {
          if (userCandidates.isEmpty()) {
            match.setAutoCorrect(ranker.shouldAutoCorrect(defaultSuggestions));
            match.setSuggestedReplacementObjects(defaultSuggestions);
          } else {
            List<SuggestedReplacement> combinedSuggestions = new ArrayList<>();
            for (String wordFromUserDict : userCandidates) {
              SuggestedReplacement s = new SuggestedReplacement(wordFromUserDict);
{'fraction_non_alphanumeric': 0.0840959293152414, 'fraction_numerical': 0.0050489113284947935, 'mean_word_length': 4.288694201084689, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1223778', 'n_tokens_mistral': 3594, 'n_tokens_neox': 3443, 'n_words': 1001}