<commit_before>/* LanguageTool, a natural language style checker 
 * Copyright (C) 2005 <NAME> (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.tools;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.regex.Pattern;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;

import org.languagetool.Language;
import org.languagetool.gui.Tools;
import org.languagetool.rules.RuleMatch;
import org.languagetool.rules.patterns.PatternRule;

/**
 * Tools for reading files etc.
 * 
 * @author <NAME>
 */
public final class StringTools {

  private static final int DEFAULT_CONTEXT_SIZE = 25;  

  /**
   * Constants for printing XML rule matches.
   */
  public static enum XmlPrintMode {
    /**
     * Normally output the rule matches by starting and
     * ending the XML output on every call.
     */
    NORMAL_XML,
    /**
     * Start XML output by printing the preamble and the
     * start of the root element.
     */
    START_XML,
    /**
     * End XML output by closing the root element.
     */
    END_XML,
    /**
     * Simply continue rule match output.
     */
    CONTINUE_XML
  }

  private static final Pattern XML_COMMENT_PATTERN = Pattern.compile("<!--.*?-->", Pattern.DOTALL);
  private static final Pattern XML_PATTERN = Pattern.compile("(?<!<)<[^<>]+>", Pattern.DOTALL);


  private StringTools() {
    // only static stuff
  }

  /**
   * Throw exception if the given string is null or empty or only whitespace.
   */
  public static void assureSet(final String s, final String varName) {
    if (s == null) {
      throw new NullPointerException(varName + " cannot be null");
    }
    if (isEmpty(s.trim())) {
      throw new IllegalArgumentException(varName
          + " cannot be empty or whitespace only");
    }
  }

  /**
   * Read a file's content.
   */
  public static String readFile(final InputStream file) throws IOException {
    return readFile(file, null);
  }

  /**
   * Read the text file using the given encoding.
   * 
   * @param file
   *          InputStream to a file to be read
   * @param encoding
   *          the file's character encoding (e.g. <code>iso-8859-1</code>)
   * @return a string with the file's content, lines separated by
   *         <code>\n</code>
   * @throws IOException
   */
  public static String readFile(final InputStream file, final String encoding)
  throws IOException {
    InputStreamReader isr = null;
    BufferedReader br = null;
    final StringBuilder sb = new StringBuilder();
    try {
      if (encoding == null) {
        isr = new InputStreamReader(file);
      } else {
        isr = new InputStreamReader(file, encoding);
      }
      br = new BufferedReader(isr);
      String line;
      while ((line = br.readLine()) != null) {
        sb.append(line);
        sb.append('\n');
      }
    } finally {
      if (br != null) {
        br.close();
      }
      if (isr != null) {
        isr.close();
      }
    }
    return sb.toString();
  }

  /**
   * Returns true if <code>str</code> is made up of all-uppercase characters
   * (ignoring characters for which no upper-/lowercase distinction exists).
   */
  public static boolean isAllUppercase(final String str) {
    return str.equals(str.toUpperCase());
  }

  /**
   * @param str - input str
   * Returns true if str is MixedCase.
   */
  public static boolean isMixedCase(final String str) {
    return !isAllUppercase(str)
    && !isCapitalizedWord(str)
    && !str.equals(str.toLowerCase());
  }

  /**
   * @param str - input string
   */
  public static boolean isCapitalizedWord(final String str) {
    if (isEmpty(str)) {
      return false;
    }
    final char firstChar = str.charAt(0);
    if (Character.isUpperCase(firstChar)) {
      return str.substring(1).equals(str.substring(1).toLowerCase());
    }
    return false;
  }

  /**
   * Whether the first character of <code>str</code> is an uppercase character.
   */
  public static boolean startsWithUppercase(final String str) {
    if (isEmpty(str)) {
      return false;
    }
    final char firstChar = str.charAt(0);
    if (Character.isUpperCase(firstChar)) {
      return true;
    }
    return false;
  }

  /**
   * Return <code>str</code> modified so that its first character is now an
   * uppercase character. If <code>str</code> starts with non-alphabetic
   * characters, such as quotes or parentheses, the first character is 
   * determined as the first alphabetic character.
   */
  public static String uppercaseFirstChar(final String str) {
      return changeFirstCharCase(str, true);
  }

  /**
   * Return <code>str</code> modified so that its first character is now an
   * lowercase character. If <code>str</code> starts with non-alphabetic
   * characters, such as quotes or parentheses, the first character is 
   * determined as the first alphabetic character.
   */
  public static String lowercaseFirstChar(final String str) {
      return changeFirstCharCase(str, false);
  }

  /**
   * Return <code>str</code> modified so that its first character is now an
   * lowercase or uppercase character, depending on <code>toUpperCase</code>.
   * If <code>str</code> starts with non-alphabetic
   * characters, such as quotes or parentheses, the first character is 
   * determined as the first alphabetic character.
   */
  private static String changeFirstCharCase(final String str, final boolean toUpperCase) {
    if (isEmpty(str)) {
      return str;
    }
    if (str.length() == 1) {
      return toUpperCase ? str.toUpperCase() : str.toLowerCase();
    }
    int pos = 0;
    final int len = str.length() - 1;
    while (!Character.isLetterOrDigit(str.charAt(pos)) && len > pos) {
      pos++;
    }
    final char firstChar = str.charAt(pos);    
    return str.substring(0, pos) 
        + (toUpperCase ? Character.toUpperCase(firstChar) : Character.toLowerCase(firstChar))
        + str.substring(pos + 1);
  }

  public static String readerToString(final Reader reader) throws IOException {
    final StringBuilder sb = new StringBuilder();
    int readBytes = 0;
    final char[] chars = new char[4000];
    while (readBytes >= 0) {
      readBytes = reader.read(chars, 0, 4000);
      if (readBytes <= 0) {
        break;
      }
      sb.append(new String(chars, 0, readBytes));
    }
    return sb.toString();
  }

  public static String streamToString(final InputStream is) throws IOException {
    final InputStreamReader isr = new InputStreamReader(is);
    try {
      return readerToString(isr);
    } finally {
      isr.close();
    }
  }

   public static String streamToString(final InputStream is, String charsetName) throws IOException {
    final InputStreamReader isr = new InputStreamReader(is, charsetName);
    try {
      return readerToString(isr);
    } finally {
      isr.close();
    }
  } 
  
  /**
   * Calls escapeHTML(String).
   */
  public static String escapeXML(final String s) {
    return escapeHTML(s);
  }

  /**
   * Escapes these characters: less than, bigger than, quote, ampersand.
   */
  public static String escapeHTML(final String s) {
    // this version is much faster than using s.replaceAll
    final StringBuilder sb = new StringBuilder();
    final int n = s.length();
    for (int i = 0; i < n; i++) {
      final char c = s.charAt(i);
      switch (c) {
      case '<':
        sb.append("&lt;");
        break;
      case '>':
        sb.append("&gt;");
        break;
      case '&':
        sb.append("&amp;");
        break;
      case '"':
        sb.append("&quot;");
        break;

      default:
        sb.append(c);
      break;
      }
    }
    return sb.toString();
  }

  /**
   * Get an XML representation of the given rule matches.
   * 
   * @param text
   *          the original text that was checked, used to get the context of the
   *          matches
   * @param contextSize
   *          the desired context size in characters
   * @deprecated Use {@link #ruleMatchesToXML(List,String,int,XmlPrintMode)} instead
   */
  public static String ruleMatchesToXML(final List<RuleMatch> ruleMatches,
      final String text, final int contextSize) {
    return ruleMatchesToXML(ruleMatches, text, contextSize, XmlPrintMode.NORMAL_XML);
  }

  /**
   * Get an XML representation of the given rule matches.
   * @param text
   *          the original text that was checked, used to get the context of the
   *          matches
   * @param contextSize
   *          the desired context size in characters
   * @param xmlMode how to print the XML
   */
  public static String ruleMatchesToXML(final List<RuleMatch> ruleMatches,
      final String text, final int contextSize, final XmlPrintMode xmlMode) {
    //
    // IMPORTANT: people rely on this format, don't change it!
    //
    final StringBuilder xml = new StringBuilder();

    if (xmlMode == XmlPrintMode.NORMAL_XML || xmlMode == XmlPrintMode.START_XML) {
      xml.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
      xml.append("<matches>\n");
    }

    for (final RuleMatch match : ruleMatches) {
      String subId = "";
      if (match.getRule() instanceof PatternRule) {
        final PatternRule pRule = (PatternRule) match.getRule();
        if (pRule.getSubId() != null) {
          subId = " subId=\"" + escapeXMLForAPIOutput(pRule.getSubId()) + "\" ";
        }
      }
      xml.append("<error" + " fromy=\"" + match.getLine() + "\"" + " fromx=\""
          + (match.getColumn() - 1) + "\"" + " toy=\"" + match.getEndLine() + "\""
          + " tox=\"" + (match.getEndColumn() - 1) + "\"" + " ruleId=\""
          + match.getRule().getId() + "\"");
      final String msg = match.getMessage().replaceAll("</?suggestion>", "'");
      xml.append(subId);
      xml.append(" msg=\"" + escapeXMLForAPIOutput(msg) + "\"");
      final String START_MARKER = "__languagetool_start_marker";
      String context = Tools.getContext(match.getFromPos(), match.getToPos(),
          text, contextSize, START_MARKER, "", false);
      xml.append(" replacements=\""
          + escapeXMLForAPIOutput(listToString(
              match.getSuggestedReplacements(), "#")) + "\"");
      // get position of error in context and remove artificial marker again:
      final int contextOffset = context.indexOf(START_MARKER);
      context = context.replaceFirst(START_MARKER, "");
      context = context.replaceAll("[\n\r]", " ");
      xml.append(" context=\"" + StringTools.escapeXML(context) + "\"");
      xml.append(" contextoffset=\"" + contextOffset + "\"");
      xml.append(" errorlength=\"" + (match.getToPos() - match.getFromPos()) + "\"");
      if (match.getRule().getUrl() != null) {
    	  xml.append(" url=\"" + match.getRule().getUrl().toString() + "\"");
      }<commit_msg>- escape XML special char before printing the URL    when using --api or else the output XML could   contain the ampersand character (invalid XML output).   This happens at least when doing:   $ echo "Wir haben eine gewisse Zeitlang gesucht." | \     java -jar dist/LanguageTool.jar -l de --api<commit_after>/* LanguageTool, a natural language style checker 
 * Copyright (C) 2005 <NAME> (http://www.danielnaber.de)
 * 
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.tools;

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.util.regex.Pattern;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;

import org.languagetool.Language;
import org.languagetool.gui.Tools;
import org.languagetool.rules.RuleMatch;
import org.languagetool.rules.patterns.PatternRule;

/**
 * Tools for reading files etc.
 * 
 * @author <NAME>
 */
public final class StringTools {

  private static final int DEFAULT_CONTEXT_SIZE = 25;  

  /**
   * Constants for printing XML rule matches.
   */
  public static enum XmlPrintMode {
    /**
     * Normally output the rule matches by starting and
     * ending the XML output on every call.
     */
    NORMAL_XML,
    /**
     * Start XML output by printing the preamble and the
     * start of the root element.
     */
    START_XML,
    /**
     * End XML output by closing the root element.
     */
    END_XML,
    /**
     * Simply continue rule match output.
     */
    CONTINUE_XML
  }

  private static final Pattern XML_COMMENT_PATTERN = Pattern.compile("<!--.*?-->", Pattern.DOTALL);
  private static final Pattern XML_PATTERN = Pattern.compile("(?<!<)<[^<>]+>", Pattern.DOTALL);


  private StringTools() {
    // only static stuff
  }

  /**
   * Throw exception if the given string is null or empty or only whitespace.
   */
  public static void assureSet(final String s, final String varName) {
    if (s == null) {
      throw new NullPointerException(varName + " cannot be null");
    }
    if (isEmpty(s.trim())) {
      throw new IllegalArgumentException(varName
          + " cannot be empty or whitespace only");
    }
  }

  /**
   * Read a file's content.
   */
  public static String readFile(final InputStream file) throws IOException {
    return readFile(file, null);
  }

  /**
   * Read the text file using the given encoding.
   * 
   * @param file
   *          InputStream to a file to be read
   * @param encoding
   *          the file's character encoding (e.g. <code>iso-8859-1</code>)
   * @return a string with the file's content, lines separated by
   *         <code>\n</code>
   * @throws IOException
   */
  public static String readFile(final InputStream file, final String encoding)
  throws IOException {
    InputStreamReader isr = null;
    BufferedReader br = null;
    final StringBuilder sb = new StringBuilder();
    try {
      if (encoding == null) {
        isr = new InputStreamReader(file);
      } else {
        isr = new InputStreamReader(file, encoding);
      }
      br = new BufferedReader(isr);
      String line;
      while ((line = br.readLine()) != null) {
        sb.append(line);
        sb.append('\n');
      }
    } finally {
      if (br != null) {
        br.close();
      }
      if (isr != null) {
        isr.close();
      }
    }
    return sb.toString();
  }

  /**
   * Returns true if <code>str</code> is made up of all-uppercase characters
   * (ignoring characters for which no upper-/lowercase distinction exists).
   */
  public static boolean isAllUppercase(final String str) {
    return str.equals(str.toUpperCase());
  }

  /**
   * @param str - input str
   * Returns true if str is MixedCase.
   */
  public static boolean isMixedCase(final String str) {
    return !isAllUppercase(str)
    && !isCapitalizedWord(str)
    && !str.equals(str.toLowerCase());
  }

  /**
   * @param str - input string
   */
  public static boolean isCapitalizedWord(final String str) {
    if (isEmpty(str)) {
      return false;
    }
    final char firstChar = str.charAt(0);
    if (Character.isUpperCase(firstChar)) {
      return str.substring(1).equals(str.substring(1).toLowerCase());
    }
    return false;
  }

  /**
   * Whether the first character of <code>str</code> is an uppercase character.
   */
  public static boolean startsWithUppercase(final String str) {
    if (isEmpty(str)) {
      return false;
    }
    final char firstChar = str.charAt(0);
    if (Character.isUpperCase(firstChar)) {
      return true;
    }
    return false;
  }

  /**
   * Return <code>str</code> modified so that its first character is now an
   * uppercase character. If <code>str</code> starts with non-alphabetic
   * characters, such as quotes or parentheses, the first character is 
   * determined as the first alphabetic character.
   */
  public static String uppercaseFirstChar(final String str) {
      return changeFirstCharCase(str, true);
  }

  /**
   * Return <code>str</code> modified so that its first character is now an
   * lowercase character. If <code>str</code> starts with non-alphabetic
   * characters, such as quotes or parentheses, the first character is 
   * determined as the first alphabetic character.
   */
  public static String lowercaseFirstChar(final String str) {
      return changeFirstCharCase(str, false);
  }

  /**
   * Return <code>str</code> modified so that its first character is now an
   * lowercase or uppercase character, depending on <code>toUpperCase</code>.
   * If <code>str</code> starts with non-alphabetic
   * characters, such as quotes or parentheses, the first character is 
   * determined as the first alphabetic character.
   */
  private static String changeFirstCharCase(final String str, final boolean toUpperCase) {
    if (isEmpty(str)) {
      return str;
    }
    if (str.length() == 1) {
      return toUpperCase ? str.toUpperCase() : str.toLowerCase();
    }
    int pos = 0;
    final int len = str.length() - 1;
    while (!Character.isLetterOrDigit(str.charAt(pos)) && len > pos) {
      pos++;
    }
    final char firstChar = str.charAt(pos);    
    return str.substring(0, pos) 
        + (toUpperCase ? Character.toUpperCase(firstChar) : Character.toLowerCase(firstChar))
        + str.substring(pos + 1);
  }

  public static String readerToString(final Reader reader) throws IOException {
    final StringBuilder sb = new StringBuilder();
    int readBytes = 0;
    final char[] chars = new char[4000];
    while (readBytes >= 0) {
      readBytes = reader.read(chars, 0, 4000);
      if (readBytes <= 0) {
        break;
      }
      sb.append(new String(chars, 0, readBytes));
    }
    return sb.toString();
  }

  public static String streamToString(final InputStream is) throws IOException {
    final InputStreamReader isr = new InputStreamReader(is);
    try {
      return readerToString(isr);
    } finally {
      isr.close();
    }
  }

   public static String streamToString(final InputStream is, String charsetName) throws IOException {
    final InputStreamReader isr = new InputStreamReader(is, charsetName);
    try {
      return readerToString(isr);
    } finally {
      isr.close();
    }
  } 
  
  /**
   * Calls escapeHTML(String).
   */
  public static String escapeXML(final String s) {
    return escapeHTML(s);
  }

  /**
   * Escapes these characters: less than, bigger than, quote, ampersand.
   */
  public static String escapeHTML(final String s) {
    // this version is much faster than using s.replaceAll
    final StringBuilder sb = new StringBuilder();
    final int n = s.length();
    for (int i = 0; i < n; i++) {
      final char c = s.charAt(i);
      switch (c) {
      case '<':
        sb.append("&lt;");
        break;
      case '>':
        sb.append("&gt;");
        break;
      case '&':
        sb.append("&amp;");
        break;
      case '"':
        sb.append("&quot;");
        break;

      default:
        sb.append(c);
      break;
      }
    }
    return sb.toString();
  }

  /**
   * Get an XML representation of the given rule matches.
   * 
   * @param text
   *          the original text that was checked, used to get the context of the
   *          matches
   * @param contextSize
   *          the desired context size in characters
   * @deprecated Use {@link #ruleMatchesToXML(List,String,int,XmlPrintMode)} instead
   */
  public static String ruleMatchesToXML(final List<RuleMatch> ruleMatches,
      final String text, final int contextSize) {
    return ruleMatchesToXML(ruleMatches, text, contextSize, XmlPrintMode.NORMAL_XML);
  }

  /**
   * Get an XML representation of the given rule matches.
   * @param text
   *          the original text that was checked, used to get the context of the
   *          matches
   * @param contextSize
   *          the desired context size in characters
   * @param xmlMode how to print the XML
   */
  public static String ruleMatchesToXML(final List<RuleMatch> ruleMatches,
      final String text, final int contextSize, final XmlPrintMode xmlMode) {
    //
    // IMPORTANT: people rely on this format, don't change it!
    //
    final StringBuilder xml = new StringBuilder();

    if (xmlMode == XmlPrintMode.NORMAL_XML || xmlMode == XmlPrintMode.START_XML) {
      xml.append("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n");
      xml.append("<matches>\n");
    }

    for (final RuleMatch match : ruleMatches) {
      String subId = "";
      if (match.getRule() instanceof PatternRule) {
        final PatternRule pRule = (PatternRule) match.getRule();
        if (pRule.getSubId() != null) {
          subId = " subId=\"" + escapeXMLForAPIOutput(pRule.getSubId()) + "\" ";
        }
      }
      xml.append("<error" + " fromy=\"" + match.getLine() + "\"" + " fromx=\""
          + (match.getColumn() - 1) + "\"" + " toy=\"" + match.getEndLine() + "\""
          + " tox=\"" + (match.getEndColumn() - 1) + "\"" + " ruleId=\""
          + match.getRule().getId() + "\"");
      final String msg = match.getMessage().replaceAll("</?suggestion>", "'");
      xml.append(subId);
      xml.append(" msg=\"" + escapeXMLForAPIOutput(msg) + "\"");
      final String START_MARKER = "__languagetool_start_marker";
      String context = Tools.getContext(match.getFromPos(), match.getToPos(),
          text, contextSize, START_MARKER, "", false);
      xml.append(" replacements=\""
          + escapeXMLForAPIOutput(listToString(
              match.getSuggestedReplacements(), "#")) + "\"");
      // get position of error in context and remove artificial marker again:
      final int contextOffset = context.indexOf(START_MARKER);
      context = context.replaceFirst(START_MARKER, "");
      context = context.replaceAll("[\n\r]", " ");
      xml.append(" context=\"" + StringTools.escapeXML(context) + "\"");
      xml.append(" contextoffset=\"" + contextOffset + "\"");
      xml.append(" errorlength=\"" + (match.getToPos() - match.getFromPos()) + "\"");
      if (match.getRule().getUrl() != null) {
          xml.append(" url=\""
              + escapeXMLForAPIOutput(match.getRule().getUrl().toString())
              + "\"");
      }
{'fraction_non_alphanumeric': 0.11973420794820462, 'fraction_numerical': 0.004429867529922903, 'mean_word_length': 2.722530521642619, 'pattern_counts': {'":': 0, '<': 79, '<?xml version=': 2, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 32}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6273016', 'n_tokens_mistral': 6978, 'n_tokens_neox': 6794, 'n_words': 2137}