<commit_before>/* LanguageTool, a natural language style checker 
 * Copyright (C) 2005 <NAME> (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.rules.patterns;

import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;
import java.util.regex.Matcher;

import junit.framework.TestCase;
import org.languagetool.AnalyzedSentence;
import org.languagetool.JLanguageTool;
import org.languagetool.Language;
import org.languagetool.TestTools;
import org.languagetool.rules.IncorrectExample;
import org.languagetool.rules.Rule;
import org.languagetool.rules.RuleMatch;

/**
 * @author <NAME>
 */
public class PatternRuleTest extends TestCase {

  // The  [^cfmnt123]\\.|\\.[^mvngl]  part is there to consider a string as a
  // regexp if and only if it is not enclosed on both sides by those characters.
  // This is to cope with Polish POS tags which contain dots without being
  // a regexp.
  private static final Pattern PROBABLE_PATTERN = Pattern.compile("(.+[+?^{}|\\[\\]].*)|(.*[+?^{}|\\[\\]].+)|(\\(.*\\))|(\\\\[^0-9].*)|[^cfmnt123]\\.|\\.[^mvngl]|(.+\\.$)");
  private static final Pattern CASE_PATTERN = Pattern.compile("\\[(.)(.)\\]");
  private static final Pattern EMPTY_DISJUNCTION = Pattern.compile("^[|]|[|][|]|[|]$");

  private static JLanguageTool langTool;

  @Override
  public void setUp() throws IOException {
    if (langTool == null) {
      langTool = new JLanguageTool(Language.ENGLISH);
    }
  }

  public void testGrammarRulesFromXML() throws IOException {
    testGrammarRulesFromXML(null);
  }

  public void testGrammarRulesFromXML2() throws IOException {
    new PatternRule("-1", Language.ENGLISH, Collections.<Element>emptyList(), "", "", "");
  }
    
  public void testGrammarRulesFromXML(Set<Language> ignoredLanguages) throws IOException {
    for (final Language lang : Language.LANGUAGES) {
      if (ignoredLanguages != null && ignoredLanguages.contains(lang)) {
        continue;
      }
      System.out.println("Running tests for " + lang.getName() + "...");
      final PatternRuleLoader ruleLoader = new PatternRuleLoader();
      final JLanguageTool languageTool = new JLanguageTool(lang);
      final JLanguageTool allRulesLanguageTool = new JLanguageTool(lang);
      allRulesLanguageTool.activateDefaultPatternRules();
      final String name = "/" + lang.getShortName() + "/grammar.xml";
      final List<PatternRule> rules = ruleLoader.getRules(JLanguageTool.getDataBroker().
              getFromRulesDirAsStream(name), name);
      warnIfRegexpSyntaxNotKosher(rules, lang);
      testGrammarRulesFromXML(rules, languageTool, allRulesLanguageTool, lang);
    }
  }

  private void testGrammarRulesFromXML(final List<PatternRule> rules,
                                       final JLanguageTool languageTool, 
                                       final JLanguageTool allRulesLanguageTool, final Language lang) throws IOException {
    final HashMap<String, PatternRule> complexRules = new HashMap<String, PatternRule>();
    for (final PatternRule rule : rules) {
      testCorrectSentences(languageTool, allRulesLanguageTool, lang, rule);
      testBadSentences(languageTool, lang, complexRules, rule);
    }
    if (!complexRules.isEmpty()) {
      final Set<String> set = complexRules.keySet();
      final List<PatternRule> badRules = new ArrayList<PatternRule>();
      for (String aSet : set) {
        final PatternRule badRule = complexRules.get(aSet);
        if (badRule != null) {
          badRule.notComplexPhrase();
          badRule.setMessage("The rule contains a phrase that never matched any incorrect example.");
          badRules.add(badRule);
        }
      }
      if (!badRules.isEmpty()) {
        testGrammarRulesFromXML(badRules, languageTool, allRulesLanguageTool, lang);
      }
    }
  }

  // TODO: probably this would be more useful for exceptions
  // instead of adding next methods to PatternRule
  // we can probably validate using XSD and specify regexes straight there
  private void warnIfRegexpSyntaxNotKosher(final List<PatternRule> rules,
      final Language lang) {
    for (final PatternRule rule : rules) {
      int i = 0;
      for (final Element element : rule.getElements()) {
        i++;

        // Check whether token value is consistent with regexp="..."
        warnIfElementNotKosher(
          element.getString(),
          element.isRegularExpression(),
          element.getCaseSensitive(),
          element.isInflected(),
          lang, rule.getId() + ":" + rule.getSubId());

        // Check postag="..." is consistent with postag_regexp="..."
        warnIfElementNotKosher(
          element.getPOStag() == null ? "" : element.getPOStag(),
          element.isPOStagRegularExpression(),
          element.getCaseSensitive(),
          false,
          lang, rule.getId() + ":" + rule.getSubId() + " (exception in POS tag) ");

        List<Element> exceptionElements = new ArrayList<Element>();
        if (element.getExceptionList() != null) {
          for (final Element exception: element.getExceptionList()) {
            // Detect useless exception or missing skip="...".
            if (exception.hasNextException() && element.getSkipNext() == 0) {
              System.err.println("The " + lang.toString() + " rule: "
                  + rule.getId() + ":" + rule.getSubId()
                  + " in token [" + i + "]"
                  + " has no skip=\"...\" and yet contains scope=\"next\""
                  + " so the exception never applies. "
                  + " Did you forget skip=\"...\"");
            }

            // Check whether exception value is consistent with regexp="..."
            // Don't check string "." since it is sometimes used as a regexp
            // and sometimes used as non regexp.
            if (!exception.getString().equals(".")) {
              warnIfElementNotKosher(
                exception.getString(),
                exception.isRegularExpression(),
                exception.getCaseSensitive(),
                exception.isInflected(),
                lang,
                rule.getId() + ":" + rule.getSubId() + " (exception in token [" + i + "]) ");
            }
            // Check postag="..." of exception is consistent with postag_regexp="..."
            warnIfElementNotKosher(
              exception.getPOStag() == null ? "" : exception.getPOStag(),
              exception.isPOStagRegularExpression(),
              exception.getCaseSensitive(),
              false,
              lang,
              rule.getId() + ":" + rule.getSubId() + " (exception in POS tag of token [" + i + "]) ");<commit_msg>- minor change in warning message in "ant test".<commit_after>/* LanguageTool, a natural language style checker 
 * Copyright (C) 2005 <NAME> (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.rules.patterns;

import java.io.IOException;
import java.util.*;
import java.util.regex.Pattern;
import java.util.regex.Matcher;

import junit.framework.TestCase;
import org.languagetool.AnalyzedSentence;
import org.languagetool.JLanguageTool;
import org.languagetool.Language;
import org.languagetool.TestTools;
import org.languagetool.rules.IncorrectExample;
import org.languagetool.rules.Rule;
import org.languagetool.rules.RuleMatch;

/**
 * @author <NAME>
 */
public class PatternRuleTest extends TestCase {

  // The  [^cfmnt123]\\.|\\.[^mvngl]  part is there to consider a string as a
  // regexp if and only if it is not enclosed on both sides by those characters.
  // This is to cope with Polish POS tags which contain dots without being
  // a regexp.
  private static final Pattern PROBABLE_PATTERN = Pattern.compile("(.+[+?^{}|\\[\\]].*)|(.*[+?^{}|\\[\\]].+)|(\\(.*\\))|(\\\\[^0-9].*)|[^cfmnt123]\\.|\\.[^mvngl]|(.+\\.$)");
  private static final Pattern CASE_PATTERN = Pattern.compile("\\[(.)(.)\\]");
  private static final Pattern EMPTY_DISJUNCTION = Pattern.compile("^[|]|[|][|]|[|]$");

  private static JLanguageTool langTool;

  @Override
  public void setUp() throws IOException {
    if (langTool == null) {
      langTool = new JLanguageTool(Language.ENGLISH);
    }
  }

  public void testGrammarRulesFromXML() throws IOException {
    testGrammarRulesFromXML(null);
  }

  public void testGrammarRulesFromXML2() throws IOException {
    new PatternRule("-1", Language.ENGLISH, Collections.<Element>emptyList(), "", "", "");
  }
    
  public void testGrammarRulesFromXML(Set<Language> ignoredLanguages) throws IOException {
    for (final Language lang : Language.LANGUAGES) {
      if (ignoredLanguages != null && ignoredLanguages.contains(lang)) {
        continue;
      }
      System.out.println("Running tests for " + lang.getName() + "...");
      final PatternRuleLoader ruleLoader = new PatternRuleLoader();
      final JLanguageTool languageTool = new JLanguageTool(lang);
      final JLanguageTool allRulesLanguageTool = new JLanguageTool(lang);
      allRulesLanguageTool.activateDefaultPatternRules();
      final String name = "/" + lang.getShortName() + "/grammar.xml";
      final List<PatternRule> rules = ruleLoader.getRules(JLanguageTool.getDataBroker().
              getFromRulesDirAsStream(name), name);
      warnIfRegexpSyntaxNotKosher(rules, lang);
      testGrammarRulesFromXML(rules, languageTool, allRulesLanguageTool, lang);
    }
  }

  private void testGrammarRulesFromXML(final List<PatternRule> rules,
                                       final JLanguageTool languageTool, 
                                       final JLanguageTool allRulesLanguageTool, final Language lang) throws IOException {
    final HashMap<String, PatternRule> complexRules = new HashMap<String, PatternRule>();
    for (final PatternRule rule : rules) {
      testCorrectSentences(languageTool, allRulesLanguageTool, lang, rule);
      testBadSentences(languageTool, lang, complexRules, rule);
    }
    if (!complexRules.isEmpty()) {
      final Set<String> set = complexRules.keySet();
      final List<PatternRule> badRules = new ArrayList<PatternRule>();
      for (String aSet : set) {
        final PatternRule badRule = complexRules.get(aSet);
        if (badRule != null) {
          badRule.notComplexPhrase();
          badRule.setMessage("The rule contains a phrase that never matched any incorrect example.");
          badRules.add(badRule);
        }
      }
      if (!badRules.isEmpty()) {
        testGrammarRulesFromXML(badRules, languageTool, allRulesLanguageTool, lang);
      }
    }
  }

  // TODO: probably this would be more useful for exceptions
  // instead of adding next methods to PatternRule
  // we can probably validate using XSD and specify regexes straight there
  private void warnIfRegexpSyntaxNotKosher(final List<PatternRule> rules,
      final Language lang) {
    for (final PatternRule rule : rules) {
      int i = 0;
      for (final Element element : rule.getElements()) {
        i++;

        // Check whether token value is consistent with regexp="..."
        warnIfElementNotKosher(
          element.getString(),
          element.isRegularExpression(),
          element.getCaseSensitive(),
          element.isInflected(),
          lang, rule.getId() + ":" + rule.getSubId());

        // Check postag="..." is consistent with postag_regexp="..."
        warnIfElementNotKosher(
          element.getPOStag() == null ? "" : element.getPOStag(),
          element.isPOStagRegularExpression(),
          element.getCaseSensitive(),
          false,
          lang, rule.getId() + ":" + rule.getSubId() + " (exception in POS tag) ");

        List<Element> exceptionElements = new ArrayList<Element>();
        if (element.getExceptionList() != null) {
          for (final Element exception: element.getExceptionList()) {
            // Detect useless exception or missing skip="...".
            if (exception.hasNextException() && element.getSkipNext() == 0) {
              System.err.println("The " + lang.toString() + " rule: "
                  + rule.getId() + ":" + rule.getSubId()
                  + " in token [" + i + "]"
                  + " has no skip=\"...\" and yet contains scope=\"next\""
                  + " so the exception never applies. "
                  + " Did you forget skip=\"...\"?");
            }

            // Check whether exception value is consistent with regexp="..."
            // Don't check string "." since it is sometimes used as a regexp
            // and sometimes used as non regexp.
            if (!exception.getString().equals(".")) {
              warnIfElementNotKosher(
                exception.getString(),
                exception.isRegularExpression(),
                exception.getCaseSensitive(),
                exception.isInflected(),
                lang,
                rule.getId() + ":" + rule.getSubId() + " (exception in token [" + i + "]) ");
            }
            // Check postag="..." of exception is consistent with postag_regexp="..."
            warnIfElementNotKosher(
              exception.getPOStag() == null ? "" : exception.getPOStag(),
              exception.isPOStagRegularExpression(),
              exception.getCaseSensitive(),
              false,
              lang,
              rule.getId() + ":" + rule.getSubId() + " (exception in POS tag of token [" + i + "]) ");
{'fraction_non_alphanumeric': 0.12146050670640834, 'fraction_numerical': 0.003929006909632841, 'mean_word_length': 2.695369211514393, 'pattern_counts': {'":': 10, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2500990', 'n_tokens_mistral': 4086, 'n_tokens_neox': 3942, 'n_words': 1160}