<commit_before><?php

/**
 * Handle request startup, before loading the environment or libraries. This
 * class bootstraps the request state up to the point where we can enter
 * Phabricator code.
 *
 * NOTE: This class MUST NOT have any dependencies. It runs before libraries
 * load.
 *
 * Rate Limiting
 * =============
 *
 * Phabricator limits the rate at which clients can request pages, and issues
 * HTTP 429 "Too Many Requests" responses if clients request too many pages too
 * quickly. Although this is not a complete defense against high-volume attacks,
 * it can  protect an install against aggressive crawlers, security scanners,
 * and some types of malicious activity.
 *
 * To perform rate limiting, each page increments a score counter for the
 * requesting user's IP. The page can give the IP more points for an expensive
 * request, or fewer for an authetnicated request.
 *
 * Score counters are kept in buckets, and writes move to a new bucket every
 * minute. After a few minutes (defined by @{method:getRateLimitBucketCount}),
 * the oldest bucket is discarded. This provides a simple mechanism for keeping
 * track of scores without needing to store, access, or read very much data.
 *
 * Users are allowed to accumulate up to 1000 points per minute, averaged across
 * all of the tracked buckets.
 *
 * @task info         Accessing Request Information
 * @task hook         Startup Hooks
 * @task apocalypse   In Case Of Apocalypse
 * @task validation   Validation
 * @task ratelimit    Rate Limiting
 * @task phases       Startup Phase Timers
 */
final class PhabricatorStartup {

  private static $startTime;
  private static $debugTimeLimit;
  private static $accessLog;
  private static $capturingOutput;
  private static $rawInput;
  private static $oldMemoryLimit;
  private static $phases;

  // TODO: For now, disable rate limiting entirely by default. We need to
  // iterate on it a bit for Conduit, some of the specific score levels, and
  // to deal with NAT'd offices.
  private static $maximumRate = 0;


/* -(  Accessing Request Information  )-------------------------------------- */


  /**
   * @task info
   */
  public static function getStartTime() {
    return self::$startTime;
  }


  /**
   * @task info
   */
  public static function getMicrosecondsSinceStart() {
    return (int)(1000000 * (microtime(true) - self::getStartTime()));
  }


  /**
   * @task info
   */
  public static function setAccessLog($access_log) {
    self::$accessLog = $access_log;
  }


  /**
   * @task info
   */
  public static function getRawInput() {
    return self::$rawInput;
  }


/* -(  Startup Hooks  )------------------------------------------------------ */


  /**
   * @param float Request start time, from `microtime(true)`.
   * @task hook
   */
  public static function didStartup($start_time) {
    self::$startTime = $start_time;

    self::$phases = array();

    self::$accessLog = null;

    static $registered;
    if (!$registered) {
      // NOTE: This protects us against multiple calls to didStartup() in the
      // same request, but also against repeated requests to the same
      // interpreter state, which we may implement in the future.
      register_shutdown_function(array(__CLASS__, 'didShutdown'));
      $registered = true;
    }

    self::setupPHP();
    self::verifyPHP();

    // If we've made it this far, the environment isn't completely broken so
    // we can switch over to relying on our own exception recovery mechanisms.
    ini_set('display_errors', 0);

    if (isset($_SERVER['REMOTE_ADDR'])) {
      self::rateLimitRequest($_SERVER['REMOTE_ADDR']);
    }

    self::normalizeInput();

    self::verifyRewriteRules();

    self::detectPostMaxSizeTriggered();

    self::beginOutputCapture();

    if (isset($_SERVER['HTTP_CONTENT_ENCODING'])) {
      $encoding = trim($_SERVER['HTTP_CONTENT_ENCODING']);
    } else {
      $encoding = null;
    }

    if ($encoding === 'gzip') {
      $source_stream = 'compress.zlib://php://input';
    } else {
      $source_stream = 'php://input';
    }

    self::$rawInput = (string)file_get_contents($source_stream);
  }


  /**
   * @task hook
   */
  public static function didShutdown() {
    $event = error_get_last();

    if (!$event) {
      return;
    }

    switch ($event['type']) {
      case E_ERROR:
      case E_PARSE:
      case E_COMPILE_ERROR:
        break;
      default:
        return;
    }
<commit_msg>Fix HTTP body decompression in PHP 5.6<commit_after><?php

/**
 * Handle request startup, before loading the environment or libraries. This
 * class bootstraps the request state up to the point where we can enter
 * Phabricator code.
 *
 * NOTE: This class MUST NOT have any dependencies. It runs before libraries
 * load.
 *
 * Rate Limiting
 * =============
 *
 * Phabricator limits the rate at which clients can request pages, and issues
 * HTTP 429 "Too Many Requests" responses if clients request too many pages too
 * quickly. Although this is not a complete defense against high-volume attacks,
 * it can  protect an install against aggressive crawlers, security scanners,
 * and some types of malicious activity.
 *
 * To perform rate limiting, each page increments a score counter for the
 * requesting user's IP. The page can give the IP more points for an expensive
 * request, or fewer for an authetnicated request.
 *
 * Score counters are kept in buckets, and writes move to a new bucket every
 * minute. After a few minutes (defined by @{method:getRateLimitBucketCount}),
 * the oldest bucket is discarded. This provides a simple mechanism for keeping
 * track of scores without needing to store, access, or read very much data.
 *
 * Users are allowed to accumulate up to 1000 points per minute, averaged across
 * all of the tracked buckets.
 *
 * @task info         Accessing Request Information
 * @task hook         Startup Hooks
 * @task apocalypse   In Case Of Apocalypse
 * @task validation   Validation
 * @task ratelimit    Rate Limiting
 * @task phases       Startup Phase Timers
 */
final class PhabricatorStartup {

  private static $startTime;
  private static $debugTimeLimit;
  private static $accessLog;
  private static $capturingOutput;
  private static $rawInput;
  private static $oldMemoryLimit;
  private static $phases;

  // TODO: For now, disable rate limiting entirely by default. We need to
  // iterate on it a bit for Conduit, some of the specific score levels, and
  // to deal with NAT'd offices.
  private static $maximumRate = 0;


/* -(  Accessing Request Information  )-------------------------------------- */


  /**
   * @task info
   */
  public static function getStartTime() {
    return self::$startTime;
  }


  /**
   * @task info
   */
  public static function getMicrosecondsSinceStart() {
    return (int)(1000000 * (microtime(true) - self::getStartTime()));
  }


  /**
   * @task info
   */
  public static function setAccessLog($access_log) {
    self::$accessLog = $access_log;
  }


  /**
   * @task info
   */
  public static function getRawInput() {
    return self::$rawInput;
  }


/* -(  Startup Hooks  )------------------------------------------------------ */


  /**
   * @param float Request start time, from `microtime(true)`.
   * @task hook
   */
  public static function didStartup($start_time) {
    self::$startTime = $start_time;

    self::$phases = array();

    self::$accessLog = null;

    static $registered;
    if (!$registered) {
      // NOTE: This protects us against multiple calls to didStartup() in the
      // same request, but also against repeated requests to the same
      // interpreter state, which we may implement in the future.
      register_shutdown_function(array(__CLASS__, 'didShutdown'));
      $registered = true;
    }

    self::setupPHP();
    self::verifyPHP();

    // If we've made it this far, the environment isn't completely broken so
    // we can switch over to relying on our own exception recovery mechanisms.
    ini_set('display_errors', 0);

    if (isset($_SERVER['REMOTE_ADDR'])) {
      self::rateLimitRequest($_SERVER['REMOTE_ADDR']);
    }

    self::normalizeInput();

    self::verifyRewriteRules();

    self::detectPostMaxSizeTriggered();

    self::beginOutputCapture();

    if (isset($_SERVER['HTTP_CONTENT_ENCODING'])) {
      $encoding = trim($_SERVER['HTTP_CONTENT_ENCODING']);
    } else {
      $encoding = null;
    }

    $input_stream = fopen('php://input', 'rb');
    if (!$input_stream) {
      self::didFatal(
        'Unable to open "php://input" to read HTTP request body.');
    }

    if ($encoding === 'gzip') {
      $ok = stream_filter_append(
        $input_stream,
        'zlib.inflate',
        STREAM_FILTER_READ,
        array(
          'window' => 30,
        ));

      if (!$ok) {
        self::didFatal(
          'Failed to append gzip inflate filter to HTTP request body input '.
          'stream.');
      }
    }

    $input_data = '';
    while (!feof($input_stream)) {
      $read_bytes = fread($input_stream, 16 * 1024);
      if ($read_bytes === false) {
        self::didFatal(
          'Failed to read input bytes from HTTP request body.');
      }
      $input_data .= $read_bytes;
    }
    fclose($input_stream);

    self::$rawInput = $input_data;
  }


  /**
   * @task hook
   */
  public static function didShutdown() {
    $event = error_get_last();

    if (!$event) {
      return;
    }

    switch ($event['type']) {
      case E_ERROR:
      case E_PARSE:
      case E_COMPILE_ERROR:
        break;
      default:
        return;
    }

{'fraction_non_alphanumeric': 0.12364357262103506, 'fraction_numerical': 0.004382303839732888, 'mean_word_length': 2.754406580493537, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3078940', 'n_tokens_mistral': 2788, 'n_tokens_neox': 2617, 'n_words': 956}