<commit_before>/* LanguageTool, a natural language style checker
 * Copyright (C) 2005 <NAME> (http://www.danielnaber.de)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.rules.patterns;

import java.io.IOException;
import java.lang.String;
import java.util.*;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import junit.framework.TestCase;

import org.languagetool.AnalyzedSentence;
import org.languagetool.JLanguageTool;
import org.languagetool.Language;
import org.languagetool.TestTools;
import org.languagetool.rules.IncorrectExample;
import org.languagetool.rules.Rule;
import org.languagetool.rules.RuleMatch;
import org.languagetool.rules.spelling.SpellingCheckRule;

/**
 * @author <NAME>
 */
public class PatternRuleTest extends TestCase {

  // A test sentence should only be a single sentence - if that's not the case it can
  // happen that rules are checked as being correct that in reality will never match.
  // This check prints a warning for affected rules, but it's disabled by default because
  // it makes the tests very slow:
  private static final boolean CHECK_WITH_SENTENCE_SPLITTING = false;

  // The  [^cfmnt123]\\.|\\.[^mvngl]  part is there to consider a string as a
  // regexp if and only if it is not enclosed on both sides by those characters.
  // This is to cope with Polish POS tags which contain dots without being
  // a regexp.
  private static final Pattern PROBABLE_PATTERN = Pattern.compile("(.+[+?^{}()|\\[\\]].*)|(.*[+?^{}()|\\[\\]].+)|(\\(.*\\))|(\\\\[^0-9].*)|[^cfmnt123]\\.|\\.[^ampvngl]|(.+\\.$)");
  private static final Pattern EMPTY_DISJUNCTION = Pattern.compile("^[|]|[|][|]|[|]$");
  private static final Pattern CHAR_SET_PATTERN = Pattern.compile("(\\(\\?-i\\))?.*(?<!\\\\)\\[^?([^\\]]+)\\]");

  private static JLanguageTool langTool;

  @Override
  public void setUp() throws IOException {
    if (langTool == null) {
      langTool = new JLanguageTool(Language.ENGLISH);
    }
  }

  public void testGrammarRulesFromXML() throws IOException {
    testGrammarRulesFromXML(null);
  }

  public void testGrammarRulesFromXML2() throws IOException {
    new PatternRule("-1", Language.ENGLISH, Collections.<Element>emptyList(), "", "", "");
  }

  public void testGrammarRulesFromXML(Set<Language> ignoredLanguages) throws IOException {
    for (final Language lang : Language.LANGUAGES) {
      if (ignoredLanguages != null && ignoredLanguages.contains(lang)) {
        continue;
      }
      System.out.println("Running tests for " + lang.getName() + "...");
      final JLanguageTool languageTool = new JLanguageTool(lang);
      if (CHECK_WITH_SENTENCE_SPLITTING) {
        languageTool.activateDefaultPatternRules();
        disableSpellingRules(languageTool);
      }
      final JLanguageTool allRulesLanguageTool = new JLanguageTool(lang);
      allRulesLanguageTool.activateDefaultPatternRules();
      final List<PatternRule> rules = new ArrayList<PatternRule>();
      for (String patternRuleFileName : lang.getRuleFileName()) {
          rules.addAll(languageTool.loadPatternRules(patternRuleFileName));
      }
      warnIfRegexpSyntaxNotKosher(rules, lang);
      testGrammarRulesFromXML(rules, languageTool, allRulesLanguageTool, lang);
    }
  }

  private void disableSpellingRules(JLanguageTool languageTool) {
    final List<Rule> allRules = languageTool.getAllRules();
    for (Rule rule : allRules) {
      if (rule instanceof SpellingCheckRule) {
        languageTool.disableRule(rule.getId());
      }
    }
  }

  private void testGrammarRulesFromXML(final List<PatternRule> rules,
                                       final JLanguageTool languageTool,
                                       final JLanguageTool allRulesLanguageTool, final Language lang) throws IOException {
    final HashMap<String, PatternRule> complexRules = new HashMap<String, PatternRule>();
    for (final PatternRule rule : rules) {
      testCorrectSentences(languageTool, allRulesLanguageTool, lang, rule);
      testBadSentences(languageTool, allRulesLanguageTool, lang, complexRules, rule);
    }
    if (!complexRules.isEmpty()) {
      final Set<String> set = complexRules.keySet();
      final List<PatternRule> badRules = new ArrayList<PatternRule>();
      for (String aSet : set) {
        final PatternRule badRule = complexRules.get(aSet);
        if (badRule != null) {
          badRule.notComplexPhrase();
          badRule.setMessage("The rule contains a phrase that never matched any incorrect example.");
          badRules.add(badRule);
        }
      }
      if (!badRules.isEmpty()) {
        testGrammarRulesFromXML(badRules, languageTool, allRulesLanguageTool, lang);
      }
    }
  }

  // TODO: probably this would be more useful for exceptions
  // instead of adding next methods to PatternRule
  // we can probably validate using XSD and specify regexes straight there
  private void warnIfRegexpSyntaxNotKosher(final List<PatternRule> rules,
      final Language lang) {
    for (final PatternRule rule : rules) {
      int i = 0;
      for (final Element element : rule.getElements()) {
        i++;

        if (element.isReferenceElement()) {
            continue;
        }

        // Check whether token value is consistent with regexp="..."
        warnIfElementNotKosher(
          element.getString(),
          element.isRegularExpression(),
          element.getCaseSensitive(),
          element.getNegation(),
          element.isInflected(),
          lang, rule.getId() + ":" + rule.getSubId(),
          i);

        // Check postag="..." is consistent with postag_regexp="..."
        warnIfElementNotKosher(
          element.getPOStag() == null ? "" : element.getPOStag(),
          element.isPOStagRegularExpression(),
          element.getCaseSensitive(),
          element.getPOSNegation(),
          false,
          lang, rule.getId() + ":" + rule.getSubId() + " (POS tag)",
          i);

        List<Element> exceptionElements = new ArrayList<Element>();
        if (element.getExceptionList() != null) {
          for (final Element exception: element.getExceptionList()) {
            // Detect useless exception or missing skip="...".
            if (exception.hasNextException() && element.getSkipNext() == 0) {
              System.err.println("The " + lang.toString() + " rule: "
                  + rule.getId() + ":" + rule.getSubId()
                  + " (exception in token [" + i + "])"
                  + " has no skip=\"...\" and yet contains scope=\"next\""
                  + " so the exception never applies. "
                  + " Did you forget skip=\"...\"?");
            }

            // Check whether exception value is consistent with regexp="..."
            // Don't check string "." since it is sometimes used as a regexp
            // and sometimes used as non regexp.
            if (!exception.getString().equals(".")) {
              warnIfElementNotKosher(
                exception.getString(),
                exception.isRegularExpression(),
                exception.getCaseSensitive(),
                exception.getNegation(),
                exception.isInflected(),
                lang,
                rule.getId() + ":" + rule.getSubId()+ " (exception in token [" + i + "])",
                i);
            }
            // Check postag="..." of exception is consistent with postag_regexp="..."
            warnIfElementNotKosher(
              exception.getPOStag() == null ? "" : exception.getPOStag(),
              exception.isPOStagRegularExpression(),
              exception.getCaseSensitive(),
              exception.getPOSNegation(),
              false,
              lang,
              rule.getId() + ":" + rule.getSubId() + " (exception in POS tag of token [" + i + "])",
              i);

            // Search for duplicate exceptions (which are useless).
            // Since there are 2 nested loops on the list of exceptions,
            // this has thus a O(n^2) complexity, where n is the number
            // of exception in a token. But n is small and it is also
            // for testing only so that's OK.
            for (final Element otherException: exceptionElements) {
              if (equalException(exception, otherException)) {
                System.err.println("The " + lang.toString() + " rule: "
                    + rule.getId() + ":" + rule.getSubId()
                    + " in token [" + i + "]"
                    + " contains duplicate exceptions with"
                    + " string=[" + exception.getString() + "]"
                    + " POS tag=[" + exception.getPOStag() + "]"
                    + " negate=[" + exception.getNegation() + "]"
                    + " POS negate=[" + exception.getPOSNegation() + "]");
                break;
              }
            }
            exceptionElements.add(exception);
          }
        }
      }
    }
  }

  /**
   * Predicate to check whether two exceptions are identical or whether
   * one exception always implies the other.
   *
   * Example #1, useless identical exceptions:
   * <exception>xx</exception><exception>xx</exception>
   *
   * Example #2, first exception implies the second exception:
   * <exception>xx</exception><exception postag="A">xx</exception>
   */
  private static boolean equalException(final Element exception1,
                                        final Element exception2)
  {
    String string1 = exception1.getString() == null ? "" : exception1.getString();
    String string2 = exception2.getString() == null ? "" : exception2.getString();
    if (!exception1.getCaseSensitive() || !exception2.getCaseSensitive()) {
      // String comparison is done case insensitive if one or both strings
      // are case insensitive, because the case insensitive one would imply
      // the case sensitive one.
      string1 = string1.toLowerCase();
      string2 = string2.toLowerCase();
    }
    if (!string1.isEmpty() && !string2.isEmpty()) {
      if (!string1.equals(string2)) {
        return false;
      }
    }

    final String posTag1 = exception1.getPOStag() == null ? "" : exception1.getPOStag();
    final String posTag2 = exception2.getPOStag() == null ? "" : exception2.getPOStag();
    if (!posTag1.isEmpty() && !posTag2.isEmpty()) {
      if (!posTag1.equals(posTag2)) {
        return false;
      }
    }

    if ( string1.isEmpty() != string2.isEmpty()
      && posTag1.isEmpty() != posTag2.isEmpty()) {
      return false;
    }

    // We should not need to check for:
    // - isCaseSensitive() since an exception without isCaseSensitive
    //   imply the one with isCaseSensitive.
    // - isInflected() since an exception with inflected="yes"
    //   implies the one without inflected="yes" if they have
    //   identical strings.
    //   without inflected="yes".
    // - isRegularExpression() since a given string is either
    //   a regexp or not.
    return exception1.getNegation() == exception2.getNegation()
        && exception1.getPOSNegation() == exception2.getPOSNegation()
        && exception1.hasNextException() == exception2.hasNextException()
        && exception1.hasPreviousException() == exception2.hasPreviousException();
  }

  private void warnIfElementNotKosher(
      final String stringValue,
      final boolean isRegularExpression,
      final boolean isCaseSensitive,
      final boolean isNegated,
      final boolean isInflected,
      final Language lang,
      final String ruleId,
      final int tokenIndex) {

    if (!isRegularExpression
        && PROBABLE_PATTERN.matcher(stringValue).find()) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], contains " + "\"" + stringValue
          + "\" that is not marked as regular expression but probably is one.");
    }

    if (isRegularExpression && stringValue.isEmpty()) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], contains an empty string " + "\""
          + stringValue + "\" that is marked as regular expression.");
    } else if (isRegularExpression
        && !PROBABLE_PATTERN.matcher(stringValue)
            .find()) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], contains " + "\"" + stringValue
          + "\" that is marked as regular expression but probably is not one.");
    }

    if (isNegated && stringValue.isEmpty()) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], marked as negated but is "
          + "empty so the negation is useless. Did you mix up "
          + "negate=\"yes\" and negate_pos=\"yes\"?");
    }
    if (isInflected && stringValue.isEmpty()) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], contains " + "\"" + stringValue
          + "\" that is marked as inflected but is empty, so the attribute is redundant.");
    }
    if (isRegularExpression && ".*".equals(stringValue)) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], marked as regular expression contains "
          + "regular expression \".*\" which is useless: "
          + "(use an empty string without regexp=\"yes\" such as <token/>)");
    }

    if (isRegularExpression) {
      final Matcher matcher = CHAR_SET_PATTERN.matcher(stringValue);
      if (matcher.find()) {
        // Remove things like \p{Punct} which are irrelevant here.
        String s = matcher.group(2).replaceAll("\\\\p\\{[^}]*\\}", "");
        // case sensitive if pattern contains (?-i).
        if (s.indexOf('|') >= 0) {
          System.err.println("The " + lang.toString() + " rule: "
             + ruleId + ", token [" + tokenIndex + "], contains | (pipe) in "
             + " regexp part [" + matcher.group(2)
             + "] which is unlikely to be correct.");
        }

        /* Disabled case insensitive check for now: it gives several errors
         * in German which are minor and debatable whether it adds value. <commit_msg>Minor change in warning given by "ant test".<commit_after>/* LanguageTool, a natural language style checker
 * Copyright (C) 2005 <NAME> (http://www.danielnaber.de)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.rules.patterns;

import java.io.IOException;
import java.lang.String;
import java.util.*;
import java.util.Arrays;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import junit.framework.TestCase;

import org.languagetool.AnalyzedSentence;
import org.languagetool.JLanguageTool;
import org.languagetool.Language;
import org.languagetool.TestTools;
import org.languagetool.rules.IncorrectExample;
import org.languagetool.rules.Rule;
import org.languagetool.rules.RuleMatch;
import org.languagetool.rules.spelling.SpellingCheckRule;

/**
 * @author <NAME>
 */
public class PatternRuleTest extends TestCase {

  // A test sentence should only be a single sentence - if that's not the case it can
  // happen that rules are checked as being correct that in reality will never match.
  // This check prints a warning for affected rules, but it's disabled by default because
  // it makes the tests very slow:
  private static final boolean CHECK_WITH_SENTENCE_SPLITTING = false;

  // The  [^cfmnt123]\\.|\\.[^mvngl]  part is there to consider a string as a
  // regexp if and only if it is not enclosed on both sides by those characters.
  // This is to cope with Polish POS tags which contain dots without being
  // a regexp.
  private static final Pattern PROBABLE_PATTERN = Pattern.compile("(.+[+?^{}()|\\[\\]].*)|(.*[+?^{}()|\\[\\]].+)|(\\(.*\\))|(\\\\[^0-9].*)|[^cfmnt123]\\.|\\.[^ampvngl]|(.+\\.$)");
  private static final Pattern EMPTY_DISJUNCTION = Pattern.compile("^[|]|[|][|]|[|]$");
  private static final Pattern CHAR_SET_PATTERN = Pattern.compile("(\\(\\?-i\\))?.*(?<!\\\\)\\[^?([^\\]]+)\\]");

  private static JLanguageTool langTool;

  @Override
  public void setUp() throws IOException {
    if (langTool == null) {
      langTool = new JLanguageTool(Language.ENGLISH);
    }
  }

  public void testGrammarRulesFromXML() throws IOException {
    testGrammarRulesFromXML(null);
  }

  public void testGrammarRulesFromXML2() throws IOException {
    new PatternRule("-1", Language.ENGLISH, Collections.<Element>emptyList(), "", "", "");
  }

  public void testGrammarRulesFromXML(Set<Language> ignoredLanguages) throws IOException {
    for (final Language lang : Language.LANGUAGES) {
      if (ignoredLanguages != null && ignoredLanguages.contains(lang)) {
        continue;
      }
      System.out.println("Running tests for " + lang.getName() + "...");
      final JLanguageTool languageTool = new JLanguageTool(lang);
      if (CHECK_WITH_SENTENCE_SPLITTING) {
        languageTool.activateDefaultPatternRules();
        disableSpellingRules(languageTool);
      }
      final JLanguageTool allRulesLanguageTool = new JLanguageTool(lang);
      allRulesLanguageTool.activateDefaultPatternRules();
      final List<PatternRule> rules = new ArrayList<PatternRule>();
      for (String patternRuleFileName : lang.getRuleFileName()) {
          rules.addAll(languageTool.loadPatternRules(patternRuleFileName));
      }
      warnIfRegexpSyntaxNotKosher(rules, lang);
      testGrammarRulesFromXML(rules, languageTool, allRulesLanguageTool, lang);
    }
  }

  private void disableSpellingRules(JLanguageTool languageTool) {
    final List<Rule> allRules = languageTool.getAllRules();
    for (Rule rule : allRules) {
      if (rule instanceof SpellingCheckRule) {
        languageTool.disableRule(rule.getId());
      }
    }
  }

  private void testGrammarRulesFromXML(final List<PatternRule> rules,
                                       final JLanguageTool languageTool,
                                       final JLanguageTool allRulesLanguageTool, final Language lang) throws IOException {
    final HashMap<String, PatternRule> complexRules = new HashMap<String, PatternRule>();
    for (final PatternRule rule : rules) {
      testCorrectSentences(languageTool, allRulesLanguageTool, lang, rule);
      testBadSentences(languageTool, allRulesLanguageTool, lang, complexRules, rule);
    }
    if (!complexRules.isEmpty()) {
      final Set<String> set = complexRules.keySet();
      final List<PatternRule> badRules = new ArrayList<PatternRule>();
      for (String aSet : set) {
        final PatternRule badRule = complexRules.get(aSet);
        if (badRule != null) {
          badRule.notComplexPhrase();
          badRule.setMessage("The rule contains a phrase that never matched any incorrect example.");
          badRules.add(badRule);
        }
      }
      if (!badRules.isEmpty()) {
        testGrammarRulesFromXML(badRules, languageTool, allRulesLanguageTool, lang);
      }
    }
  }

  // TODO: probably this would be more useful for exceptions
  // instead of adding next methods to PatternRule
  // we can probably validate using XSD and specify regexes straight there
  private void warnIfRegexpSyntaxNotKosher(final List<PatternRule> rules,
      final Language lang) {
    for (final PatternRule rule : rules) {
      int i = 0;
      for (final Element element : rule.getElements()) {
        i++;

        if (element.isReferenceElement()) {
            continue;
        }

        // Check whether token value is consistent with regexp="..."
        warnIfElementNotKosher(
          element.getString(),
          element.isRegularExpression(),
          element.getCaseSensitive(),
          element.getNegation(),
          element.isInflected(),
          lang, rule.getId() + ":" + rule.getSubId(),
          i);

        // Check postag="..." is consistent with postag_regexp="..."
        warnIfElementNotKosher(
          element.getPOStag() == null ? "" : element.getPOStag(),
          element.isPOStagRegularExpression(),
          element.getCaseSensitive(),
          element.getPOSNegation(),
          false,
          lang, rule.getId() + ":" + rule.getSubId() + " (POS tag)",
          i);

        List<Element> exceptionElements = new ArrayList<Element>();
        if (element.getExceptionList() != null) {
          for (final Element exception: element.getExceptionList()) {
            // Detect useless exception or missing skip="...".
            if (exception.hasNextException() && element.getSkipNext() == 0) {
              System.err.println("The " + lang.toString() + " rule: "
                  + rule.getId() + ":" + rule.getSubId()
                  + " (exception in token [" + i + "])"
                  + " has no skip=\"...\" and yet contains scope=\"next\""
                  + " so the exception never applies. "
                  + " Did you forget skip=\"...\"?");
            }

            // Check whether exception value is consistent with regexp="..."
            // Don't check string "." since it is sometimes used as a regexp
            // and sometimes used as non regexp.
            if (!exception.getString().equals(".")) {
              warnIfElementNotKosher(
                exception.getString(),
                exception.isRegularExpression(),
                exception.getCaseSensitive(),
                exception.getNegation(),
                exception.isInflected(),
                lang,
                rule.getId() + ":" + rule.getSubId()+ " (exception in token [" + i + "])",
                i);
            }
            // Check postag="..." of exception is consistent with postag_regexp="..."
            warnIfElementNotKosher(
              exception.getPOStag() == null ? "" : exception.getPOStag(),
              exception.isPOStagRegularExpression(),
              exception.getCaseSensitive(),
              exception.getPOSNegation(),
              false,
              lang,
              rule.getId() + ":" + rule.getSubId() + " (exception in POS tag of token [" + i + "])",
              i);

            // Search for duplicate exceptions (which are useless).
            // Since there are 2 nested loops on the list of exceptions,
            // this has thus a O(n^2) complexity, where n is the number
            // of exception in a token. But n is small and it is also
            // for testing only so that's OK.
            for (final Element otherException: exceptionElements) {
              if (equalException(exception, otherException)) {
                System.err.println("The " + lang.toString() + " rule: "
                    + rule.getId() + ":" + rule.getSubId()
                    + " in token [" + i + "]"
                    + " contains duplicate exceptions with"
                    + " string=[" + exception.getString() + "]"
                    + " POS tag=[" + exception.getPOStag() + "]"
                    + " negate=[" + exception.getNegation() + "]"
                    + " POS negate=[" + exception.getPOSNegation() + "]");
                break;
              }
            }
            exceptionElements.add(exception);
          }
        }
      }
    }
  }

  /**
   * Predicate to check whether two exceptions are identical or whether
   * one exception always implies the other.
   *
   * Example #1, useless identical exceptions:
   * <exception>xx</exception><exception>xx</exception>
   *
   * Example #2, first exception implies the second exception:
   * <exception>xx</exception><exception postag="A">xx</exception>
   */
  private static boolean equalException(final Element exception1,
                                        final Element exception2)
  {
    String string1 = exception1.getString() == null ? "" : exception1.getString();
    String string2 = exception2.getString() == null ? "" : exception2.getString();
    if (!exception1.getCaseSensitive() || !exception2.getCaseSensitive()) {
      // String comparison is done case insensitive if one or both strings
      // are case insensitive, because the case insensitive one would imply
      // the case sensitive one.
      string1 = string1.toLowerCase();
      string2 = string2.toLowerCase();
    }
    if (!string1.isEmpty() && !string2.isEmpty()) {
      if (!string1.equals(string2)) {
        return false;
      }
    }

    final String posTag1 = exception1.getPOStag() == null ? "" : exception1.getPOStag();
    final String posTag2 = exception2.getPOStag() == null ? "" : exception2.getPOStag();
    if (!posTag1.isEmpty() && !posTag2.isEmpty()) {
      if (!posTag1.equals(posTag2)) {
        return false;
      }
    }

    if ( string1.isEmpty() != string2.isEmpty()
      && posTag1.isEmpty() != posTag2.isEmpty()) {
      return false;
    }

    // We should not need to check for:
    // - isCaseSensitive() since an exception without isCaseSensitive
    //   imply the one with isCaseSensitive.
    // - isInflected() since an exception with inflected="yes"
    //   implies the one without inflected="yes" if they have
    //   identical strings.
    //   without inflected="yes".
    // - isRegularExpression() since a given string is either
    //   a regexp or not.
    return exception1.getNegation() == exception2.getNegation()
        && exception1.getPOSNegation() == exception2.getPOSNegation()
        && exception1.hasNextException() == exception2.hasNextException()
        && exception1.hasPreviousException() == exception2.hasPreviousException();
  }

  private void warnIfElementNotKosher(
      final String stringValue,
      final boolean isRegularExpression,
      final boolean isCaseSensitive,
      final boolean isNegated,
      final boolean isInflected,
      final Language lang,
      final String ruleId,
      final int tokenIndex) {

    if (!isRegularExpression
        && PROBABLE_PATTERN.matcher(stringValue).find()) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], contains " + "\"" + stringValue
          + "\" that is not marked as regular expression but probably is one.");
    }

    if (isRegularExpression && stringValue.isEmpty()) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], contains an empty string " + "\""
          + stringValue + "\" that is marked as regular expression.");
    } else if (isRegularExpression
        && !PROBABLE_PATTERN.matcher(stringValue)
            .find()) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], contains " + "\"" + stringValue
          + "\" that is marked as regular expression but probably is not one.");
    }

    if (isNegated && stringValue.isEmpty()) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], marked as negated but is "
          + "empty so the negation is useless. Did you mix up "
          + "negate=\"yes\" and negate_pos=\"yes\"?");
    }
    if (isInflected && stringValue.isEmpty()) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], contains " + "\"" + stringValue
          + "\" that is marked as inflected but is empty, so the attribute is redundant.");
    }
    if (isRegularExpression && ".*".equals(stringValue)) {
      System.err.println("The " + lang.toString() + " rule: "
          + ruleId + ", token [" + tokenIndex + "], marked as regular expression contains "
          + "regular expression \".*\" which is useless: "
          + "(use an empty string without regexp=\"yes\" such as <token/>)");
    }

    if (isRegularExpression) {
      final Matcher matcher = CHAR_SET_PATTERN.matcher(stringValue);
      if (matcher.find()) {
        // Remove things like \p{Punct} which are irrelevant here.
        String s = matcher.group(2).replaceAll("\\\\p\\{[^}]*\\}", "");
        // case sensitive if pattern contains (?-i).
        if (s.indexOf('|') >= 0) {
          System.err.println("The " + lang.toString() + " rule: "
             + ruleId + ", token [" + tokenIndex + "], contains | (pipe) in "
             + " regexp bracket expression [" + matcher.group(2)
             + "] which is unlikely to be correct.");
        }

        /* Disabled case insensitive check for now: it gives several errors
         * in German which are minor and debatable whether it adds value. 
{'fraction_non_alphanumeric': 0.1257573221757322, 'fraction_numerical': 0.005087866108786611, 'mean_word_length': 2.468307406547481, 'pattern_counts': {'":': 12, '<': 55, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '21617', 'n_tokens_mistral': 8246, 'n_tokens_neox': 8006, 'n_words': 2318}