<commit_before>/*
 * $Id: ProxyManager.java,v 1.49 2011-08-09 03:59:23 tlipkis Exp $
 */

/*

Copyright (c) 2000-2003 Board of Trustees of Leland Stanford Jr. University,
all rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
STANFORD UNIVERSITY BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Except as contained in this notice, the name of Stanford University shall not
be used in advertising or otherwise to promote the sale, use or other dealings
in this Software without prior written authorization from Stanford University.

*/

package org.lockss.proxy;

import java.util.*;
import org.mortbay.http.*;

import org.lockss.app.*;
import org.lockss.util.*;
import org.lockss.util.urlconn.*;
import org.lockss.config.*;
import org.lockss.daemon.*;
import org.lockss.jetty.*;
import org.lockss.servlet.*;

/** LOCKSS proxy manager, starts main proxy.
 */
public class ProxyManager extends BaseProxyManager {
  public static final String SERVER_NAME = "Proxy";
  private static Logger log = Logger.getLogger("Proxy");

  public static final String PREFIX = Configuration.PREFIX + "proxy.";
  public static final String PARAM_START = PREFIX + "start";
  public static final boolean DEFAULT_START = true;

  public static final String PARAM_PORT = PREFIX + "port";
  public static final int DEFAULT_PORT = 9090;

  /** List of IP addresses to which to bind listen socket.  If not set,
   * server listens on all interfaces.  All listeners must be on the same
   * port, given by the <tt>port</tt> parameter.  Changing this requires
   * daemon restart. */
  public static final String PARAM_BIND_ADDRS = PREFIX + "bindAddrs";

  public static final String IP_ACCESS_PREFIX = PREFIX + "access.ip.";
  public static final String PARAM_IP_INCLUDE = IP_ACCESS_PREFIX + "include";
  public static final String PARAM_IP_EXCLUDE = IP_ACCESS_PREFIX + "exclude";
  public static final String PARAM_IP_PLATFORM_SUBNET =
    IP_ACCESS_PREFIX + IpAccessControl.SUFFIX_PLATFORM_ACCESS;
  public static final String PARAM_LOG_FORBIDDEN =
    IP_ACCESS_PREFIX + "logForbidden";
  public static final boolean DEFAULT_LOG_FORBIDDEN = true;

  /** The log level at which to log all proxy accesses.  To normally log
   * all content accesses (proxy or ServeContent), set to <tt>info</tt>.
   * To disable set to <tt>none</tt>. */
  static final String PARAM_ACCESS_LOG_LEVEL = PREFIX + "accessLogLevel";
  static final String DEFAULT_ACCESS_LOG_LEVEL = "info";

  /** If true, successive accesses to recently accessed content on the
   * cache does not trigger a request to the publisher */
  static final String PARAM_URL_CACHE_ENABLED = PREFIX + "urlCache.enabled";
  static final boolean DEFAULT_URL_CACHE_ENABLED = false;

  /** Duration during which successive accesses to a recently accessed URL
   * does not trigger a request to the publisher */
  static final String PARAM_URL_CACHE_DURATION = PREFIX + "urlCache.duration";
  static final long DEFAULT_URL_CACHE_DURATION = 1 * Constants.HOUR;

  /** The amount of time after which the "down" status of a host is
   * cleared, so that a request will once again cause a connection
   * attempt */
  static final String PARAM_HOST_DOWN_RETRY = PREFIX + "hostDownRetry";
  static final long DEFAULT_HOST_DOWN_RETRY = 10 * Constants.MINUTE;

  /** Cookie policy in effect for proxy connections to origin server. {@see
   * org.lockss.util.Constants}.  Set to <code>default</code> to use
   * daemon's default cookie policy, {@see
   * HttpClientUrlConnection#PARAM_COOKIE_POLICY}. */
  static final String PARAM_COOKIE_POLICY = PREFIX + "cookiePolicy";
  static final String DEFAULT_COOKIE_POLICY = "ignore";
  /** Set {@see #PARAM_COOKIE_POLICY} to this to have proxy use daemon's
   * default cookie policy, {@see
   * HttpClientUrlConnection#PARAM_COOKIE_POLICY}. */
  public static final String COOKIE_POLICY_DEFAULT = "default";

  public static final int HOST_DOWN_NO_CACHE_ACTION_504 = 1;
  public static final int HOST_DOWN_NO_CACHE_ACTION_QUICK = 2;
  public static final int HOST_DOWN_NO_CACHE_ACTION_NORMAL = 3;
  public static final int HOST_DOWN_NO_CACHE_ACTION_DEFAULT =
    HOST_DOWN_NO_CACHE_ACTION_NORMAL;

  /** For hosts believed to be down, this controls what we do with requests
   * for content not in the cache.  <ul><li><code>1</code>: Return
   * an immediate error.  <li><code>2</code>: Attempt to contact using
   * the quick-timeout connection pool.  <li><code>3</code>: Attempt
   * to contact normally.</ul>
   */
  static final String PARAM_HOST_DOWN_ACTION = PREFIX + "hostDownAction";
  static final int DEFAULT_HOST_DOWN_ACTION =
    HOST_DOWN_NO_CACHE_ACTION_DEFAULT;

  /** A semicolon-separated list of response codes for which the manifest
   * index should <b>not</b> be generated, or <code>all</code> to disable
   * manifest indices for all server responses.  (Index is always generated
   * if connection error (timeout, refuse, etc.) */
  static final String PARAM_NO_MANIFEST_INDEX_RESPONSES =
    PREFIX + "noManifestIndexResponses";
  static final String DEFAULT_NO_MANIFEST_INDEX_RESPONSES = "301;302;303;307";

  /** A semicolon-separated list of HTTP methods that the proxy should
   * never allow */
  static final String PARAM_DISALLOWED_METHODS =
    PREFIX + "disallowedMethods";
  static final String DEFAULT_DISALLOWED_METHODS = HttpRequest.__CONNECT;

  public static final String PARAM_PROXY_MAX_TOTAL_CONN =
    PREFIX + "connectionPool.max";
  public static final int DEFAULT_PROXY_MAX_TOTAL_CONN = 15;

  public static final String PARAM_PROXY_MAX_CONN_PER_HOST =
    PREFIX + "connectionPool.maxPerHost";
  public static final int DEFAULT_PROXY_MAX_CONN_PER_HOST = 2;

  // See comments regarding connect timeouts in HttpClientUrlConnection
  public static final String PARAM_PROXY_CONNECT_TIMEOUT =
    PREFIX + "timeout.connect";
  public static final long DEFAULT_PROXY_CONNECT_TIMEOUT =
    1 * Constants.MINUTE;
  public static final String PARAM_PROXY_DATA_TIMEOUT =
    PREFIX + "timeout.data";
  public static final long DEFAULT_PROXY_DATA_TIMEOUT =
    30 * Constants.MINUTE;

  public static final String PARAM_PROXY_QUICK_CONNECT_TIMEOUT =
    PREFIX + "quickTimeout.connect";
  public static final long DEFAULT_PROXY_QUICK_CONNECT_TIMEOUT =
    15 * Constants.SECOND;
  public static final String PARAM_PROXY_QUICK_DATA_TIMEOUT =
    PREFIX + "quickTimeout.data";
  public static final long DEFAULT_PROXY_QUICK_DATA_TIMEOUT =
    5  * Constants.MINUTE;

  public static final String PARAM_CLOSE_IDLE_CONNECTION_INTERVAL =
    PREFIX + "closeIdleConnections.interval";
  public static final long DEFAULT_CLOSE_IDLE_CONNECTION_INTERVAL =
    10 * Constants.MINUTE;

  public static final String PARAM_CLOSE_IDLE_CONNECTION_IDLE_TIME =
    PREFIX + "closeIdleConnections.idleTime";
  public static final long DEFAULT_CLOSE_IDLE_CONNECTION_IDLE_TIME =
    10 * Constants.MINUTE;

  /** Content Re-Writing Support - GIF to PNG */<commit_msg>Increased default connection pool sizes to accommodate default Proxy  and ServeContent thread pool sizes.<commit_after>/*
 * $Id: ProxyManager.java,v 1.50 2012-10-30 00:12:03 tlipkis Exp $
 */

/*

Copyright (c) 2000-2003 Board of Trustees of Leland Stanford Jr. University,
all rights reserved.

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
STANFORD UNIVERSITY BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR
IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Except as contained in this notice, the name of Stanford University shall not
be used in advertising or otherwise to promote the sale, use or other dealings
in this Software without prior written authorization from Stanford University.

*/

package org.lockss.proxy;

import java.util.*;
import org.mortbay.http.*;

import org.lockss.app.*;
import org.lockss.util.*;
import org.lockss.util.urlconn.*;
import org.lockss.config.*;
import org.lockss.daemon.*;
import org.lockss.jetty.*;
import org.lockss.servlet.*;

/** LOCKSS proxy manager, starts main proxy.
 */
public class ProxyManager extends BaseProxyManager {
  public static final String SERVER_NAME = "Proxy";
  private static Logger log = Logger.getLogger("Proxy");

  public static final String PREFIX = Configuration.PREFIX + "proxy.";
  public static final String PARAM_START = PREFIX + "start";
  public static final boolean DEFAULT_START = true;

  public static final String PARAM_PORT = PREFIX + "port";
  public static final int DEFAULT_PORT = 9090;

  /** List of IP addresses to which to bind listen socket.  If not set,
   * server listens on all interfaces.  All listeners must be on the same
   * port, given by the <tt>port</tt> parameter.  Changing this requires
   * daemon restart. */
  public static final String PARAM_BIND_ADDRS = PREFIX + "bindAddrs";

  public static final String IP_ACCESS_PREFIX = PREFIX + "access.ip.";
  public static final String PARAM_IP_INCLUDE = IP_ACCESS_PREFIX + "include";
  public static final String PARAM_IP_EXCLUDE = IP_ACCESS_PREFIX + "exclude";
  public static final String PARAM_IP_PLATFORM_SUBNET =
    IP_ACCESS_PREFIX + IpAccessControl.SUFFIX_PLATFORM_ACCESS;
  public static final String PARAM_LOG_FORBIDDEN =
    IP_ACCESS_PREFIX + "logForbidden";
  public static final boolean DEFAULT_LOG_FORBIDDEN = true;

  /** The log level at which to log all proxy accesses.  To normally log
   * all content accesses (proxy or ServeContent), set to <tt>info</tt>.
   * To disable set to <tt>none</tt>. */
  static final String PARAM_ACCESS_LOG_LEVEL = PREFIX + "accessLogLevel";
  static final String DEFAULT_ACCESS_LOG_LEVEL = "info";

  /** If true, successive accesses to recently accessed content on the
   * cache does not trigger a request to the publisher */
  static final String PARAM_URL_CACHE_ENABLED = PREFIX + "urlCache.enabled";
  static final boolean DEFAULT_URL_CACHE_ENABLED = false;

  /** Duration during which successive accesses to a recently accessed URL
   * does not trigger a request to the publisher */
  static final String PARAM_URL_CACHE_DURATION = PREFIX + "urlCache.duration";
  static final long DEFAULT_URL_CACHE_DURATION = 1 * Constants.HOUR;

  /** The amount of time after which the "down" status of a host is
   * cleared, so that a request will once again cause a connection
   * attempt */
  static final String PARAM_HOST_DOWN_RETRY = PREFIX + "hostDownRetry";
  static final long DEFAULT_HOST_DOWN_RETRY = 10 * Constants.MINUTE;

  /** Cookie policy in effect for proxy connections to origin server. {@see
   * org.lockss.util.Constants}.  Set to <code>default</code> to use
   * daemon's default cookie policy, {@see
   * HttpClientUrlConnection#PARAM_COOKIE_POLICY}. */
  static final String PARAM_COOKIE_POLICY = PREFIX + "cookiePolicy";
  static final String DEFAULT_COOKIE_POLICY = "ignore";
  /** Set {@see #PARAM_COOKIE_POLICY} to this to have proxy use daemon's
   * default cookie policy, {@see
   * HttpClientUrlConnection#PARAM_COOKIE_POLICY}. */
  public static final String COOKIE_POLICY_DEFAULT = "default";

  public static final int HOST_DOWN_NO_CACHE_ACTION_504 = 1;
  public static final int HOST_DOWN_NO_CACHE_ACTION_QUICK = 2;
  public static final int HOST_DOWN_NO_CACHE_ACTION_NORMAL = 3;
  public static final int HOST_DOWN_NO_CACHE_ACTION_DEFAULT =
    HOST_DOWN_NO_CACHE_ACTION_NORMAL;

  /** For hosts believed to be down, this controls what we do with requests
   * for content not in the cache.  <ul><li><code>1</code>: Return
   * an immediate error.  <li><code>2</code>: Attempt to contact using
   * the quick-timeout connection pool.  <li><code>3</code>: Attempt
   * to contact normally.</ul>
   */
  static final String PARAM_HOST_DOWN_ACTION = PREFIX + "hostDownAction";
  static final int DEFAULT_HOST_DOWN_ACTION =
    HOST_DOWN_NO_CACHE_ACTION_DEFAULT;

  /** A semicolon-separated list of response codes for which the manifest
   * index should <b>not</b> be generated, or <code>all</code> to disable
   * manifest indices for all server responses.  (Index is always generated
   * if connection error (timeout, refuse, etc.) */
  static final String PARAM_NO_MANIFEST_INDEX_RESPONSES =
    PREFIX + "noManifestIndexResponses";
  static final String DEFAULT_NO_MANIFEST_INDEX_RESPONSES = "301;302;303;307";

  /** A semicolon-separated list of HTTP methods that the proxy should
   * never allow */
  static final String PARAM_DISALLOWED_METHODS =
    PREFIX + "disallowedMethods";
  static final String DEFAULT_DISALLOWED_METHODS = HttpRequest.__CONNECT;

  /** Maximum total outgoing connections to publisher sites, from Proxy or
   * ServeContent.  Should be larger than max total Proxy or ServeContent
   * threads */
  public static final String PARAM_PROXY_MAX_TOTAL_CONN =
    PREFIX + "connectionPool.max";
  public static final int DEFAULT_PROXY_MAX_TOTAL_CONN = 20;

  /** Maximum per-host outgoing connections to publisher sites, from Proxy
   * or ServeContent.  Should be larger than max total Proxy or
   * ServeContent threads */
  public static final String PARAM_PROXY_MAX_CONN_PER_HOST =
    PREFIX + "connectionPool.maxPerHost";
  public static final int DEFAULT_PROXY_MAX_CONN_PER_HOST = 10;

  // See comments regarding connect timeouts in HttpClientUrlConnection
  public static final String PARAM_PROXY_CONNECT_TIMEOUT =
    PREFIX + "timeout.connect";
  public static final long DEFAULT_PROXY_CONNECT_TIMEOUT =
    1 * Constants.MINUTE;
  public static final String PARAM_PROXY_DATA_TIMEOUT =
    PREFIX + "timeout.data";
  public static final long DEFAULT_PROXY_DATA_TIMEOUT =
    30 * Constants.MINUTE;

  public static final String PARAM_PROXY_QUICK_CONNECT_TIMEOUT =
    PREFIX + "quickTimeout.connect";
  public static final long DEFAULT_PROXY_QUICK_CONNECT_TIMEOUT =
    15 * Constants.SECOND;
  public static final String PARAM_PROXY_QUICK_DATA_TIMEOUT =
    PREFIX + "quickTimeout.data";
  public static final long DEFAULT_PROXY_QUICK_DATA_TIMEOUT =
    5  * Constants.MINUTE;

  public static final String PARAM_CLOSE_IDLE_CONNECTION_INTERVAL =
    PREFIX + "closeIdleConnections.interval";
  public static final long DEFAULT_CLOSE_IDLE_CONNECTION_INTERVAL =
    10 * Constants.MINUTE;

  public static final String PARAM_CLOSE_IDLE_CONNECTION_IDLE_TIME =
    PREFIX + "closeIdleConnections.idleTime";
  public static final long DEFAULT_CLOSE_IDLE_CONNECTION_IDLE_TIME =
    10 * Constants.MINUTE;

  /** Content Re-Writing Support - GIF to PNG */
{'fraction_non_alphanumeric': 0.06855088632701128, 'fraction_numerical': 0.008243461014007686, 'mean_word_length': 4.501193317422435, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6424367', 'n_tokens_mistral': 4865, 'n_tokens_neox': 4643, 'n_words': 1719}