<commit_before><?php

/**
 * Created by PhpStorm.
 * User: Exodus4D
 * Date: 27.03.2017
 * Time: 16:06
 */

namespace Exodus4D\ESI\Lib;

class WebClient extends \Web {

    const CACHE_KEY_ERROR_LIMIT                 = 'CACHED_ERROR_LIMIT';
    const CACHE_KEY_LOGGABLE_LIMIT              = 'CACHED_LOGGABLE_LIMIT';

    const ERROR_STATUS_LOG                      = 'HTTP %s: \'%s\' | url: %s \'%s\'%s';
    const ERROR_RESOURCE_LEGACY                 = 'Resource: %s has been marked as legacy. url: %s | header: %s';
    const ERROR_RESOURCE_DEPRECATED             = 'Resource: %s has been marked as deprecated. url: %s | header: %s';
    const ERROR_LIMIT_CRITICAL                  = 'Error rate reached critical amount. url: %s | errorCount: %s | errorRemainCount: %s';
    const ERROR_LIMIT_EXCEEDED                  = 'Error rate limit exceeded! We are blocked for (%s seconds)';
    const DEBUG_URI_BLOCKED                     = 'Debug request blocked. Error limit exceeded. url: %s blocked for %2ss';

    const REQUEST_METHODS                       = ['GET', 'POST', 'PUT', 'DELETE'];

    // error limits ---------------------------------------------------------------------------------------------------
    // ESI calls will return special headers in case a client hits a "error limit" for a single endpoint

    // log error when this error count is reached for a single API endpoint in the current error window
    const ERROR_COUNT_MAX_URL                   = 30;

    // log error if less then this errors remain in current error window (all endpoints)
    const ERROR_COUNT_REMAIN_TOTAL              = 10;

    // max number of CREST curls for a single endpoint until giving up...
    // ->this is because CREST is not very stable
    const RETRY_COUNT_MAX                       = 2;

    // loggable limits ------------------------------------------------------------------------------------------------
    // ESI endpoints that return warning headers (e.g. "resource_legacy", "resource_deprecated") will get logged
    // To prevent big file I/O on these log files, errors get "throttled" and not all of them get logged

    // Time interval used for error inspection (seconds)
    const LOGGABLE_COUNT_INTERVAL               = 60;

    // Log first "2" errors that occur for an endpoint within "60" (LOGGABLE_COUNT_INTERVAL) seconds interval
    const LOGGABLE_COUNT_MAX_URL                = 2;


    /**
     * debugLevel used for internal error/warning logging
     * @var int
     */
    protected $debugLevel                       = 0;

    public function __construct(int $debugLevel = 0){
        $this->debugLevel = $debugLevel;
    }

    /**
     * parse array with HTTP header data
     * @param array $headers
     * @return array
     */
    protected function parseHeaders(array $headers = []): array {
        $parsedHeaders = [];
        foreach($headers as $header){
            $parts = explode(':', $header, 2);
            $parsedHeaders[strtolower(trim($parts[0]))] = isset($parts[1]) ? trim($parts[1]) :  '';
        }
        return $parsedHeaders;
    }

    /**
     * @param array $headers
     * @return int
     */
    protected function getStatusCodeFromHeaders(array $headers = []): int {
        $statusCode = 0;
        foreach($headers as $key => $value){
            if(preg_match('/http\/1\.\d (\d{3}?)/i', $key, $matches)){
                $statusCode = (int)$matches[1];
                break;
            }
        }
        return $statusCode;
    }

    /**
     * get HTTP status type from HTTP status code (e.g. 404 )> 'err_client')
     * @param int $statusCode
     * @return string
     */
    protected function getStatusType(int $statusCode): string{
        $typeLevel = (int)substr($statusCode, 0, 1);
        switch($typeLevel){
            case 1:
                $statusType = 'info';
                break;
            case 2:
                $statusType = 'ok';
                break;
            case 3:
                $statusType = 'redirect';
                break;
            case 4:
                $statusType = 'err_client';
                break;
            case 5:
                $statusType = 'err_server';
                break;
            default:
                $statusType = 'unknown';
        }

        return $statusType;
    }

    /**
     * @param int $code
     * @param string $method
     * @param string $url
     * @param null $responseBody
     * @return string
     */
    protected function getErrorMessageFromJsonResponse(int $code, string $method, string $url, $responseBody = null):string {
        $message = empty($responseBody->message) ?  @constant('Base::HTTP_' . $code) : $responseBody->message;
        $body = !is_null($responseBody) ? ' | body: ' . print_r($responseBody, true) : '';

        return sprintf(self::ERROR_STATUS_LOG, $code, $message, $method, $url, $body);
    }

    /**
     * get Logger obj for given status type
     * @param string $statusType
     * @return \Log
     */
    public function getLogger(string $statusType): \Log{
        switch($statusType){
            case 'err_server':
                $logfile = 'esi_error_server';
                break;
            case 'err_client':
                $logfile = 'esi_error_client';
                break;
            case 'resource_legacy':
                $logfile = 'esi_resource_legacy';
                break;
            case 'resource_deprecated':
                $logfile = 'esi_resource_deprecated';
                break;
            default:
                $logfile = 'esi_error_unknown';
        }
        return new \Log($logfile . '.log');
    }

    /**
     * check response headers for warnings/errors and log them
     * @param array $headers
     * @param string $url
     */
    protected function checkResponseHeaders(array $headers, string $url){
        $statusCode = $this->getStatusCodeFromHeaders($headers);
var_dump($headers);
        // check ESI warnings -----------------------------------------------------------------------------------------
        // extract ESI related headers
        $warningHeaders = array_filter($headers, function($key){
            return preg_match('/^warning/i', $key);
        }, ARRAY_FILTER_USE_KEY);

        if(count($warningHeaders)){
            // get "normalized" url path without params/placeholders
            $urlPath = $this->getNormalizedUrlPath($url);
            foreach($warningHeaders as $key => $value){
                if( preg_match('/^199/i', $value) && $this->isLoggable('legacy', $url) ){
                    $this->getLogger('resource_legacy')->write(sprintf(self::ERROR_RESOURCE_LEGACY, $urlPath, $url, $value));
                }
                if( preg_match('/^299/i', $value) && $this->isLoggable('deprecated', $url) ){
                     $this->getLogger('resource_deprecated')->write(sprintf(self::ERROR_RESOURCE_DEPRECATED, $urlPath, $url, $value));
                }
            }
        }

        // check ESI error limits -------------------------------------------------------------------------------------
        if($statusCode >= 400 && $statusCode <= 599){
            // extract ESI related headers
            $esiHeaders = array_filter($headers, function($key){
                return preg_match('/^x-esi-/i', $key);
            }, ARRAY_FILTER_USE_KEY);<commit_msg>- added error log bundling for "deprecated" and "legacy" response headers<commit_after><?php

/**
 * Created by PhpStorm.
 * User: Exodus4D
 * Date: 27.03.2017
 * Time: 16:06
 */

namespace Exodus4D\ESI\Lib;

class WebClient extends \Web {

    const CACHE_KEY_ERROR_LIMIT                 = 'CACHED_ERROR_LIMIT';
    const CACHE_KEY_LOGGABLE_LIMIT              = 'CACHED_LOGGABLE_LIMIT';

    const ERROR_STATUS_LOG                      = 'HTTP %s: \'%s\' | url: %s \'%s\'%s';
    const ERROR_RESOURCE_LEGACY                 = 'Resource: %s has been marked as legacy. url: %s | header: %s';
    const ERROR_RESOURCE_DEPRECATED             = 'Resource: %s has been marked as deprecated. url: %s | header: %s';
    const ERROR_LIMIT_CRITICAL                  = 'Error rate reached critical amount. url: %s | errorCount: %s | errorRemainCount: %s';
    const ERROR_LIMIT_EXCEEDED                  = 'Error rate limit exceeded! We are blocked for (%s seconds)';
    const DEBUG_URI_BLOCKED                     = 'Debug request blocked. Error limit exceeded. url: %s blocked for %2ss';

    const REQUEST_METHODS                       = ['GET', 'POST', 'PUT', 'DELETE'];

    // error limits ---------------------------------------------------------------------------------------------------
    // ESI calls will return special headers in case a client hits a "error limit" for a single endpoint

    // log error when this error count is reached for a single API endpoint in the current error window
    const ERROR_COUNT_MAX_URL                   = 30;

    // log error if less then this errors remain in current error window (all endpoints)
    const ERROR_COUNT_REMAIN_TOTAL              = 10;

    // max number of CREST curls for a single endpoint until giving up...
    // ->this is because CREST is not very stable
    const RETRY_COUNT_MAX                       = 2;

    // loggable limits ------------------------------------------------------------------------------------------------
    // ESI endpoints that return warning headers (e.g. "resource_legacy", "resource_deprecated") will get logged
    // To prevent big file I/O on these log files, errors get "throttled" and not all of them get logged

    // Time interval used for error inspection (seconds)
    const LOGGABLE_COUNT_INTERVAL               = 60;

    // Log first "2" errors that occur for an endpoint within "60" (LOGGABLE_COUNT_INTERVAL) seconds interval
    const LOGGABLE_COUNT_MAX_URL                = 2;


    /**
     * debugLevel used for internal error/warning logging
     * @var int
     */
    protected $debugLevel                       = 0;

    public function __construct(int $debugLevel = 0){
        $this->debugLevel = $debugLevel;
    }

    /**
     * parse array with HTTP header data
     * @param array $headers
     * @return array
     */
    protected function parseHeaders(array $headers = []): array {
        $parsedHeaders = [];
        foreach($headers as $header){
            $parts = explode(':', $header, 2);
            $parsedHeaders[strtolower(trim($parts[0]))] = isset($parts[1]) ? trim($parts[1]) :  '';
        }
        return $parsedHeaders;
    }

    /**
     * @param array $headers
     * @return int
     */
    protected function getStatusCodeFromHeaders(array $headers = []): int {
        $statusCode = 0;
        foreach($headers as $key => $value){
            if(preg_match('/http\/1\.\d (\d{3}?)/i', $key, $matches)){
                $statusCode = (int)$matches[1];
                break;
            }
        }
        return $statusCode;
    }

    /**
     * get HTTP status type from HTTP status code (e.g. 404 )> 'err_client')
     * @param int $statusCode
     * @return string
     */
    protected function getStatusType(int $statusCode): string{
        $typeLevel = (int)substr($statusCode, 0, 1);
        switch($typeLevel){
            case 1:
                $statusType = 'info';
                break;
            case 2:
                $statusType = 'ok';
                break;
            case 3:
                $statusType = 'redirect';
                break;
            case 4:
                $statusType = 'err_client';
                break;
            case 5:
                $statusType = 'err_server';
                break;
            default:
                $statusType = 'unknown';
        }

        return $statusType;
    }

    /**
     * @param int $code
     * @param string $method
     * @param string $url
     * @param null $responseBody
     * @return string
     */
    protected function getErrorMessageFromJsonResponse(int $code, string $method, string $url, $responseBody = null):string {
        $message = empty($responseBody->message) ?  @constant('Base::HTTP_' . $code) : $responseBody->message;
        $body = !is_null($responseBody) ? ' | body: ' . print_r($responseBody, true) : '';

        return sprintf(self::ERROR_STATUS_LOG, $code, $message, $method, $url, $body);
    }

    /**
     * get Logger obj for given status type
     * @param string $statusType
     * @return \Log
     */
    public function getLogger(string $statusType): \Log{
        switch($statusType){
            case 'err_server':
                $logfile = 'esi_error_server';
                break;
            case 'err_client':
                $logfile = 'esi_error_client';
                break;
            case 'resource_legacy':
                $logfile = 'esi_resource_legacy';
                break;
            case 'resource_deprecated':
                $logfile = 'esi_resource_deprecated';
                break;
            default:
                $logfile = 'esi_error_unknown';
        }
        return new \Log($logfile . '.log');
    }

    /**
     * check response headers for warnings/errors and log them
     * @param array $headers
     * @param string $url
     */
    protected function checkResponseHeaders(array $headers, string $url){
        $statusCode = $this->getStatusCodeFromHeaders($headers);
var_dump($headers);
        // check ESI warnings -----------------------------------------------------------------------------------------
        // extract ESI related headers
        $warningHeaders = array_filter($headers, function($key){
            return preg_match('/^warning/i', $key);
        }, ARRAY_FILTER_USE_KEY);
        var_dump($warningHeaders);
        if(count($warningHeaders)){
            // get "normalized" url path without params/placeholders
            $urlPath = $this->getNormalizedUrlPath($url);
            foreach($warningHeaders as $key => $value){
                if( preg_match('/^199/i', $value) && $this->isLoggable('legacy', $url) ){
                    $this->getLogger('resource_legacy')->write(sprintf(self::ERROR_RESOURCE_LEGACY, $urlPath, $url, $value));
                }
                if( preg_match('/^299/i', $value) && $this->isLoggable('deprecated', $url) ){
                     $this->getLogger('resource_deprecated')->write(sprintf(self::ERROR_RESOURCE_DEPRECATED, $urlPath, $url, $value));
                }
            }
        }

        // check ESI error limits -------------------------------------------------------------------------------------
        if($statusCode >= 400 && $statusCode <= 599){
            // extract ESI related headers
            $esiHeaders = array_filter($headers, function($key){
                return preg_match('/^x-esi-/i', $key);
            }, ARRAY_FILTER_USE_KEY);
{'fraction_non_alphanumeric': 0.16008358047991372, 'fraction_numerical': 0.007818819088703155, 'mean_word_length': 1.6723703170028819, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '75276', 'n_tokens_mistral': 4069, 'n_tokens_neox': 3879, 'n_words': 1094}