<commit_before>/* LanguageTool, a natural language style checker
 * Copyright (C) 2011 <NAME> (http://www.danielnaber.de)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.server;

import java.io.*;
import java.net.*;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;

import org.apache.commons.lang.StringUtils;
import org.jetbrains.annotations.Nullable;
import org.languagetool.JLanguageTool;
import org.languagetool.Language;
import org.languagetool.Languages;
import org.languagetool.gui.Configuration;
import org.languagetool.language.LanguageIdentifier;
import org.languagetool.rules.RuleMatch;
import org.languagetool.rules.bitext.BitextRule;
import org.languagetool.tools.RuleAsXmlSerializer;
import org.languagetool.tools.StringTools;
import org.languagetool.tools.Tools;

import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;

import static org.languagetool.tools.StringTools.escapeForXmlContent;

class LanguageToolHttpHandler implements HttpHandler {

  private static final String CONTENT_TYPE_VALUE = "text/xml; charset=UTF-8";
  private static final String ENCODING = "utf-8";
  private static final int CONTEXT_SIZE = 40; // characters

  private static int handleCount = 0;

  private final Set<String> allowedIps;  
  private final boolean verbose;
  private final boolean internalServer;
  private final RequestLimiter requestLimiter;
  private final LinkedBlockingQueue<Runnable> workQueue;
  private final ExecutorService executorService;
  private final LanguageIdentifier identifier;
  private final Set<String> ownIps;

  private long maxCheckTimeMillis = -1;
  private int maxTextLength = Integer.MAX_VALUE;
  private String allowOriginUrl;
  private boolean afterTheDeadlineMode;
  private Language afterTheDeadlineLanguage;
  private File languageModelDir;
  private int maxWorkQueueSize;
  private boolean trustXForwardForHeader = false;
  
  /**
   * Create an instance. Call {@link #shutdown()} when done.
   * @param verbose print the input text in case of exceptions
   * @param allowedIps set of IPs that may connect or <tt>null</tt> to allow any IP
   * @param requestLimiter may be null
   */
  LanguageToolHttpHandler(boolean verbose, Set<String> allowedIps, boolean internal, RequestLimiter requestLimiter, LinkedBlockingQueue<Runnable> workQueue) {
    this.verbose = verbose;
    this.allowedIps = allowedIps;
    this.internalServer = internal;
    this.requestLimiter = requestLimiter;
    this.workQueue = workQueue;
    this.executorService = Executors.newCachedThreadPool();
    this.ownIps = getServersOwnIps();
    this.identifier = new LanguageIdentifier();
  }

  /** @since 2.6 */
  void shutdown() {
    executorService.shutdownNow();
  }

  void setMaxTextLength(int maxTextLength) {
    this.maxTextLength = maxTextLength;
  }

  /**
   * Maximum time allowed per check in milliseconds. If the checking takes longer, it will stop with
   * an exception. Use {@code -1} for no limit.
   * @since 2.6
   */
  void setMaxCheckTimeMillis(long maxCheckTimeMillis) {
    this.maxCheckTimeMillis = maxCheckTimeMillis;
  }

  /**
   * Set to {@code true} if this is running behind a (reverse) proxy which
   * sets the 'X-forwarded-for' HTTP header. The last IP address (but not local IP addresses)
   * in that header will then be used for enforcing a request limitation.
   * @since 2.8
   */
  void setTrustXForwardForHeader(boolean trustXForwardForHeader) {
    this.trustXForwardForHeader = trustXForwardForHeader;
  }

  /**
   * Value to set as the "Access-Control-Allow-Origin" http header. Use {@code null}
   * to not return that header at all. Use {@code *} to run a server that any other web site
   * can use from Javascript/Ajax (search Cross-origin resource sharing (CORS) for details).
   */
  void setAllowOriginUrl(String allowOriginUrl) {
    this.allowOriginUrl = allowOriginUrl;
  }

  /** @since 2.7 */
  void setAfterTheDeadlineMode(Language defaultLanguage) {
    System.out.println("Running in After the Deadline mode, default language: " + defaultLanguage);
    this.afterTheDeadlineMode = true;
    this.afterTheDeadlineLanguage = defaultLanguage;
  }

  /** @since 2.7 */
  void setLanguageModel(File languageModelDir) {
    this.languageModelDir = languageModelDir;
  }

  /**
   * @param size maximum queue size - if the queue is larger, the user will get an error. Use {@code 0} for no limit.
   * @since 2.9
   */
  void setMaxWorkQueueSize(int size) {
    if (size < 0) {
      throw new IllegalArgumentException("Max queue size must be >= 0: " + size);
    }
    this.maxWorkQueueSize = size;
  }

  @Override
  public void handle(HttpExchange httpExchange) throws IOException {
    synchronized (this) {
      handleCount++;
    }
    String text = null;
    try {
      final URI requestedUri = httpExchange.getRequestURI();
      final String origAddress = httpExchange.getRemoteAddress().getAddress().getHostAddress();
      final String realAddressOrNull = getRealRemoteAddressOrNull(httpExchange);
      final String remoteAddress = realAddressOrNull != null ? realAddressOrNull : origAddress;
      // According to the Javadoc, "Closing an exchange without consuming all of the request body is
      // not an error but may make the underlying TCP connection unusable for following exchanges.",
      // so we consume the request now, even before checking for request limits:
      final Map<String, String> parameters = getRequestQuery(httpExchange, requestedUri);
      if (requestLimiter != null && !requestLimiter.isAccessOkay(remoteAddress)) {
        final String errorMessage = "Error: Access from " + remoteAddress +
                " denied - too many requests. Allowed maximum requests: " + requestLimiter.getRequestLimit() +
                " requests per " + requestLimiter.getRequestLimitPeriodInSeconds() + " seconds";
        sendError(httpExchange, HttpURLConnection.HTTP_FORBIDDEN, errorMessage);
        print(errorMessage);
        return;
      }
      if (maxWorkQueueSize != 0 && workQueue.size() > maxWorkQueueSize) {
        String response = "Error: There are currently too many parallel requests. Please try again later.";
        print(response + " Queue size: " + workQueue.size() + ", maximum size: " + maxWorkQueueSize);
        sendError(httpExchange, HttpURLConnection.HTTP_UNAVAILABLE, "Error: " + response);
        return;
      }
      if (allowedIps == null || allowedIps.contains(origAddress)) {
        if (requestedUri.getRawPath().endsWith("/Languages")) {
          // request type: list known languages
          printListOfLanguages(httpExchange);
        } else {
          // request type: text checking
          if (afterTheDeadlineMode) {
            text = parameters.get("data");
            if (text == null) {
              throw new IllegalArgumentException("Missing 'data' parameter");
            }
            text = text.replaceAll("</p>", "\n\n").replaceAll("<.*?>", "");  // clean up HTML, position changes don't matter for AtD
          } else {
            text = parameters.get("text");
            if (text == null) {
              throw new IllegalArgumentException("Missing 'text' parameter");
            }
          }
          checkText(text, httpExchange, parameters);
        }
      } else {
        final String errorMessage = "Error: Access from " + StringTools.escapeXML(origAddress) + " denied";
        sendError(httpExchange, HttpURLConnection.HTTP_FORBIDDEN, errorMessage);
        throw new RuntimeException(errorMessage);
      }
    } catch (Exception e) {
      print("An error has occurred. Stacktrace follows:", System.err);
      if (verbose && text != null) {
        print("Exception was caused by this text (" + text.length() + " chars, showing up to 500):\n" +
                StringUtils.abbreviate(text, 500), System.err);
      }
      //noinspection CallToPrintStackTrace
      e.printStackTrace();
      String response;
      int errorCode;
      if (e instanceof TextTooLongException) {
        errorCode = HttpURLConnection.HTTP_ENTITY_TOO_LARGE;
        response = e.getMessage();
      } else if (e.getCause() != null && e.getCause() instanceof TimeoutException) {
        errorCode = HttpURLConnection.HTTP_UNAVAILABLE;
        response = "Checking took longer than " + maxCheckTimeMillis/1000 + " seconds, which is this server's limit. " +
                   "Please make sure you have selected the proper language or consider submitting a shorter text.";
      } else {
        response = Tools.getFullStackTrace(e);
        errorCode = HttpURLConnection.HTTP_INTERNAL_ERROR;
      }
      sendError(httpExchange, errorCode, "Error: " + response);
    } finally {
      synchronized (this) {
        handleCount--;
      }
      httpExchange.close();
    }
  }

  private Set<String> getServersOwnIps() {
    Set<String> ownIps = new HashSet<>();
    try {
      Enumeration e = NetworkInterface.getNetworkInterfaces();
      while (e.hasMoreElements()) {
        NetworkInterface netInterface = (NetworkInterface) e.nextElement();
        Enumeration addresses = netInterface.getInetAddresses();
        while (addresses.hasMoreElements()) {<commit_msg>don't get own IPs unless it's really needed, seems to be very slow (>10sec) on some Windows machines<commit_after>/* LanguageTool, a natural language style checker
 * Copyright (C) 2011 <NAME> (http://www.danielnaber.de)
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */
package org.languagetool.server;

import java.io.*;
import java.net.*;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.*;

import org.apache.commons.lang.StringUtils;
import org.jetbrains.annotations.Nullable;
import org.languagetool.JLanguageTool;
import org.languagetool.Language;
import org.languagetool.Languages;
import org.languagetool.gui.Configuration;
import org.languagetool.language.LanguageIdentifier;
import org.languagetool.rules.RuleMatch;
import org.languagetool.rules.bitext.BitextRule;
import org.languagetool.tools.RuleAsXmlSerializer;
import org.languagetool.tools.StringTools;
import org.languagetool.tools.Tools;

import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;

import static org.languagetool.tools.StringTools.escapeForXmlContent;

class LanguageToolHttpHandler implements HttpHandler {

  private static final String CONTENT_TYPE_VALUE = "text/xml; charset=UTF-8";
  private static final String ENCODING = "utf-8";
  private static final int CONTEXT_SIZE = 40; // characters

  private static int handleCount = 0;

  private final Set<String> allowedIps;  
  private final boolean verbose;
  private final boolean internalServer;
  private final RequestLimiter requestLimiter;
  private final LinkedBlockingQueue<Runnable> workQueue;
  private final ExecutorService executorService;
  private final LanguageIdentifier identifier;

  private long maxCheckTimeMillis = -1;
  private int maxTextLength = Integer.MAX_VALUE;
  private String allowOriginUrl;
  private boolean afterTheDeadlineMode;
  private Language afterTheDeadlineLanguage;
  private File languageModelDir;
  private int maxWorkQueueSize;
  private boolean trustXForwardForHeader = false;
  private Set<String> ownIps;
  
  /**
   * Create an instance. Call {@link #shutdown()} when done.
   * @param verbose print the input text in case of exceptions
   * @param allowedIps set of IPs that may connect or <tt>null</tt> to allow any IP
   * @param requestLimiter may be null
   */
  LanguageToolHttpHandler(boolean verbose, Set<String> allowedIps, boolean internal, RequestLimiter requestLimiter, LinkedBlockingQueue<Runnable> workQueue) {
    this.verbose = verbose;
    this.allowedIps = allowedIps;
    this.internalServer = internal;
    this.requestLimiter = requestLimiter;
    this.workQueue = workQueue;
    this.executorService = Executors.newCachedThreadPool();
    this.identifier = new LanguageIdentifier();
  }

  /** @since 2.6 */
  void shutdown() {
    executorService.shutdownNow();
  }

  void setMaxTextLength(int maxTextLength) {
    this.maxTextLength = maxTextLength;
  }

  /**
   * Maximum time allowed per check in milliseconds. If the checking takes longer, it will stop with
   * an exception. Use {@code -1} for no limit.
   * @since 2.6
   */
  void setMaxCheckTimeMillis(long maxCheckTimeMillis) {
    this.maxCheckTimeMillis = maxCheckTimeMillis;
  }

  /**
   * Set to {@code true} if this is running behind a (reverse) proxy which
   * sets the 'X-forwarded-for' HTTP header. The last IP address (but not local IP addresses)
   * in that header will then be used for enforcing a request limitation.
   * @since 2.8
   */
  void setTrustXForwardForHeader(boolean trustXForwardForHeader) {
    this.trustXForwardForHeader = trustXForwardForHeader;
    if (trustXForwardForHeader) {
      this.ownIps = getServersOwnIps();
    }
  }

  /**
   * Value to set as the "Access-Control-Allow-Origin" http header. Use {@code null}
   * to not return that header at all. Use {@code *} to run a server that any other web site
   * can use from Javascript/Ajax (search Cross-origin resource sharing (CORS) for details).
   */
  void setAllowOriginUrl(String allowOriginUrl) {
    this.allowOriginUrl = allowOriginUrl;
  }

  /** @since 2.7 */
  void setAfterTheDeadlineMode(Language defaultLanguage) {
    System.out.println("Running in After the Deadline mode, default language: " + defaultLanguage);
    this.afterTheDeadlineMode = true;
    this.afterTheDeadlineLanguage = defaultLanguage;
  }

  /** @since 2.7 */
  void setLanguageModel(File languageModelDir) {
    this.languageModelDir = languageModelDir;
  }

  /**
   * @param size maximum queue size - if the queue is larger, the user will get an error. Use {@code 0} for no limit.
   * @since 2.9
   */
  void setMaxWorkQueueSize(int size) {
    if (size < 0) {
      throw new IllegalArgumentException("Max queue size must be >= 0: " + size);
    }
    this.maxWorkQueueSize = size;
  }

  @Override
  public void handle(HttpExchange httpExchange) throws IOException {
    synchronized (this) {
      handleCount++;
    }
    String text = null;
    try {
      final URI requestedUri = httpExchange.getRequestURI();
      final String origAddress = httpExchange.getRemoteAddress().getAddress().getHostAddress();
      final String realAddressOrNull = getRealRemoteAddressOrNull(httpExchange);
      final String remoteAddress = realAddressOrNull != null ? realAddressOrNull : origAddress;
      // According to the Javadoc, "Closing an exchange without consuming all of the request body is
      // not an error but may make the underlying TCP connection unusable for following exchanges.",
      // so we consume the request now, even before checking for request limits:
      final Map<String, String> parameters = getRequestQuery(httpExchange, requestedUri);
      if (requestLimiter != null && !requestLimiter.isAccessOkay(remoteAddress)) {
        final String errorMessage = "Error: Access from " + remoteAddress +
                " denied - too many requests. Allowed maximum requests: " + requestLimiter.getRequestLimit() +
                " requests per " + requestLimiter.getRequestLimitPeriodInSeconds() + " seconds";
        sendError(httpExchange, HttpURLConnection.HTTP_FORBIDDEN, errorMessage);
        print(errorMessage);
        return;
      }
      if (maxWorkQueueSize != 0 && workQueue.size() > maxWorkQueueSize) {
        String response = "Error: There are currently too many parallel requests. Please try again later.";
        print(response + " Queue size: " + workQueue.size() + ", maximum size: " + maxWorkQueueSize);
        sendError(httpExchange, HttpURLConnection.HTTP_UNAVAILABLE, "Error: " + response);
        return;
      }
      if (allowedIps == null || allowedIps.contains(origAddress)) {
        if (requestedUri.getRawPath().endsWith("/Languages")) {
          // request type: list known languages
          printListOfLanguages(httpExchange);
        } else {
          // request type: text checking
          if (afterTheDeadlineMode) {
            text = parameters.get("data");
            if (text == null) {
              throw new IllegalArgumentException("Missing 'data' parameter");
            }
            text = text.replaceAll("</p>", "\n\n").replaceAll("<.*?>", "");  // clean up HTML, position changes don't matter for AtD
          } else {
            text = parameters.get("text");
            if (text == null) {
              throw new IllegalArgumentException("Missing 'text' parameter");
            }
          }
          checkText(text, httpExchange, parameters);
        }
      } else {
        final String errorMessage = "Error: Access from " + StringTools.escapeXML(origAddress) + " denied";
        sendError(httpExchange, HttpURLConnection.HTTP_FORBIDDEN, errorMessage);
        throw new RuntimeException(errorMessage);
      }
    } catch (Exception e) {
      print("An error has occurred. Stacktrace follows:", System.err);
      if (verbose && text != null) {
        print("Exception was caused by this text (" + text.length() + " chars, showing up to 500):\n" +
                StringUtils.abbreviate(text, 500), System.err);
      }
      //noinspection CallToPrintStackTrace
      e.printStackTrace();
      String response;
      int errorCode;
      if (e instanceof TextTooLongException) {
        errorCode = HttpURLConnection.HTTP_ENTITY_TOO_LARGE;
        response = e.getMessage();
      } else if (e.getCause() != null && e.getCause() instanceof TimeoutException) {
        errorCode = HttpURLConnection.HTTP_UNAVAILABLE;
        response = "Checking took longer than " + maxCheckTimeMillis/1000 + " seconds, which is this server's limit. " +
                   "Please make sure you have selected the proper language or consider submitting a shorter text.";
      } else {
        response = Tools.getFullStackTrace(e);
        errorCode = HttpURLConnection.HTTP_INTERNAL_ERROR;
      }
      sendError(httpExchange, errorCode, "Error: " + response);
    } finally {
      synchronized (this) {
        handleCount--;
      }
      httpExchange.close();
    }
  }

  // Call only if really needed, seems to be slow on some Windows machines.
  private Set<String> getServersOwnIps() {
    Set<String> ownIps = new HashSet<>();
    try {
      Enumeration e = NetworkInterface.getNetworkInterfaces();
      while (e.hasMoreElements()) {
        NetworkInterface netInterface = (NetworkInterface) e.nextElement();
        Enumeration addresses = netInterface.getInetAddresses();
        while (addresses.hasMoreElements()) {
{'fraction_non_alphanumeric': 0.09250859791656282, 'fraction_numerical': 0.005083985445845586, 'mean_word_length': 3.3807860262008735, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '538022', 'n_tokens_mistral': 5597, 'n_tokens_neox': 5425, 'n_words': 1723}