<commit_before><?php

namespace Kibo\Phast\Filters\HTML\Composite;

use Kibo\Phast\Filters\HTML\HTMLPageContext;
use Kibo\Phast\Filters\HTML\HTMLStreamFilter;
use Kibo\Phast\Logging\LoggingTrait;
use Kibo\Phast\Parsing\HTML\PCRETokenizer;
use Kibo\Phast\ValueObjects\URL;

class Filter {
    use LoggingTrait;

    /**
     * @var integer
     */
    private $maxBufferSizeToApply;

    /**
     * @var URL
     */
    private $baseUrl;

    /**
     * @var HTMLStreamFilter[]
     */
    private $filters = [];

    private $timings = [];

    /**
     * Filter constructor.
     * @param int $maxBufferSizeToApply
     * @param URL $baseUrl
     */
    public function __construct($maxBufferSizeToApply, URL $baseUrl) {
        $this->maxBufferSizeToApply = $maxBufferSizeToApply;
        $this->baseUrl = $baseUrl;
    }


    /**
     * @param string $buffer
     * @return string
     */
    public function apply($buffer) {
        $time_start = microtime(true);

        if (strlen($buffer) > $this->maxBufferSizeToApply) {
            $this->logger()->info(
                'Buffer exceeds max. size ({buffersize} bytes). Not applying',
                ['buffersize' => $this->maxBufferSizeToApply]
            );
            return $buffer;
        }


        $pattern = "~
            ^
            \s* (<\?xml[^>]*>)?
            \s* (<!doctype\s+html[^>]*>)?
            (\s* <!--(.*?)-->)*
            \s* <html (?! [^>]* \s ( amp | ⚡ ) [\s=>] )
            .*
            ( </body> | </html> )
        ~isx";

        if (!preg_match($pattern, $buffer)) {
            $this->logger()->info('Buffer doesn\'t look like html! Not applying filters');
            return $buffer;
        }

        try {
            $output = $this->tryToApply($buffer, $time_start);
        } catch (\Exception $e) {
            $this->logger()->critical(
                'Phast: CompositeHTMLFilter: {exception} Msg: {message}, Code: {code}, File: {file}, Line: {line}',
                [
                    'exception' => get_class($e),
                    'message' => $e->getMessage(),
                    'code' => $e->getCode(),
                    'file' => $e->getFile(),
                    'line' => $e->getLine()
                ]
            );
            $output = $buffer;
        }
        return $output;
    }

    public function addHTMLFilter(HTMLStreamFilter $filter) {
        $this->filters[] = $filter;
    }

    private function tryToApply($buffer, $time_start) {

        $context = new HTMLPageContext($this->baseUrl);
        $elements = (new PCRETokenizer())->tokenize($buffer);

        foreach ($this->filters as $filter) {
            $this->logger()->info('Starting {filter}', ['filter' => get_class($filter)]);
            $this->time(get_class($filter), function () use ($filter, $context, &$elements) {
                $elements = $filter->transformElements($elements, $context);
            });
        }

        $output = $this->time('Serialization', function () use ($elements) {
            $output = '';
            foreach ($elements as $element) {
                $output .= $element;
            }
            return $output;
        });

        $time_delta = microtime(true) - $time_start;

        $time_accounted = 0.;
        $log = "Page automatically optimized by Phast\n\n";
        arsort($this->timings);
        foreach ($this->timings as $cls => $time) {
            $cls = str_replace('Kibo\Phast\Filters\HTML\\', '', $cls);
            $log .= sprintf("      % -43s % 4dms\n", $cls, $time*1000);
            $time_accounted += $time;
        }
        $log .= "\n";
        $log .= sprintf("      % 43s % 4dms\n", '(other)', ($time_delta - $time_accounted)*1000);
        $log .= sprintf("      % 43s % 4dms\n", '(total)', $time_delta*1000);

        $output .= '<script>window.console&&console.log(' . json_encode($log) . ')</script>';
        $this->logger()->info($log);
        return $output;
    }

    private function time($label, callable $cb) {
        $start = microtime(true);
        $returned = $cb();
        $delta = microtime(true) - $start;
        $this->timings[$label] = $delta;
        $this->logger()->info('Finished {label} in {time}', ['label' => $label, 'time' => $delta]);
        return $returned;
    }

}
<commit_msg>Output profiling in a group<commit_after><?php

namespace Kibo\Phast\Filters\HTML\Composite;

use Kibo\Phast\Filters\HTML\HTMLPageContext;
use Kibo\Phast\Filters\HTML\HTMLStreamFilter;
use Kibo\Phast\Logging\LoggingTrait;
use Kibo\Phast\Parsing\HTML\PCRETokenizer;
use Kibo\Phast\ValueObjects\URL;

class Filter {
    use LoggingTrait;

    /**
     * @var integer
     */
    private $maxBufferSizeToApply;

    /**
     * @var URL
     */
    private $baseUrl;

    /**
     * @var HTMLStreamFilter[]
     */
    private $filters = [];

    private $timings = [];

    /**
     * Filter constructor.
     * @param int $maxBufferSizeToApply
     * @param URL $baseUrl
     */
    public function __construct($maxBufferSizeToApply, URL $baseUrl) {
        $this->maxBufferSizeToApply = $maxBufferSizeToApply;
        $this->baseUrl = $baseUrl;
    }


    /**
     * @param string $buffer
     * @return string
     */
    public function apply($buffer) {
        $time_start = microtime(true);

        if (strlen($buffer) > $this->maxBufferSizeToApply) {
            $this->logger()->info(
                'Buffer exceeds max. size ({buffersize} bytes). Not applying',
                ['buffersize' => $this->maxBufferSizeToApply]
            );
            return $buffer;
        }


        $pattern = "~
            ^
            \s* (<\?xml[^>]*>)?
            \s* (<!doctype\s+html[^>]*>)?
            (\s* <!--(.*?)-->)*
            \s* <html (?! [^>]* \s ( amp | ⚡ ) [\s=>] )
            .*
            ( </body> | </html> )
        ~isx";

        if (!preg_match($pattern, $buffer)) {
            $this->logger()->info('Buffer doesn\'t look like html! Not applying filters');
            return $buffer;
        }

        try {
            $output = $this->tryToApply($buffer, $time_start);
        } catch (\Exception $e) {
            $this->logger()->critical(
                'Phast: CompositeHTMLFilter: {exception} Msg: {message}, Code: {code}, File: {file}, Line: {line}',
                [
                    'exception' => get_class($e),
                    'message' => $e->getMessage(),
                    'code' => $e->getCode(),
                    'file' => $e->getFile(),
                    'line' => $e->getLine()
                ]
            );
            $output = $buffer;
        }
        return $output;
    }

    public function addHTMLFilter(HTMLStreamFilter $filter) {
        $this->filters[] = $filter;
    }

    private function tryToApply($buffer, $time_start) {

        $context = new HTMLPageContext($this->baseUrl);
        $elements = (new PCRETokenizer())->tokenize($buffer);

        foreach ($this->filters as $filter) {
            $this->logger()->info('Starting {filter}', ['filter' => get_class($filter)]);
            $this->time(get_class($filter), function () use ($filter, $context, &$elements) {
                $elements = $filter->transformElements($elements, $context);
            });
        }

        $output = $this->time('Serialization', function () use ($elements) {
            $output = '';
            foreach ($elements as $element) {
                $output .= $element;
            }
            return $output;
        });

        $time_delta = microtime(true) - $time_start;

        $time_accounted = 0.;
        arsort($this->timings);
        foreach ($this->timings as $cls => $time) {
            $cls = str_replace('Kibo\Phast\Filters\HTML\\', '', $cls);
            $log .= sprintf("% -43s % 4dms\n", $cls, $time*1000);
            $time_accounted += $time;
        }
        $log .= "\n";
        $log .= sprintf("% 43s % 4dms\n", '(other)', ($time_delta - $time_accounted)*1000);
        $log .= sprintf("% 43s % 4dms\n", '(total)', $time_delta*1000);

        $output .= '<script>try{';
        $output .= 'console.group(' . json_encode("[Phast] Server-side performance metrics") . ');';
        $output .= 'console.log(' . json_encode($log) . ');console.groupEnd();';
        $output .= '}catch(e){}</script>';
        $this->logger()->info($log);
        return $output;
    }

    private function time($label, callable $cb) {
        $start = microtime(true);
        $returned = $cb();
        $delta = microtime(true) - $start;
        $this->timings[$label] = $delta;
        $this->logger()->info('Finished {label} in {time}', ['label' => $label, 'time' => $delta]);
        return $returned;
    }

}

{'fraction_non_alphanumeric': 0.17684887459807075, 'fraction_numerical': 0.005052824988516307, 'mean_word_length': 1.8828202581926514, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 109, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1351223', 'n_tokens_mistral': 2729, 'n_tokens_neox': 2617, 'n_words': 520}