<commit_before><?php

namespace Kibo\Phast\Parsing\HTML;

use Kibo\Phast\Exceptions\RuntimeException;
use Kibo\Phast\Parsing\HTML\HTMLStreamElements\ClosingTag;
use Kibo\Phast\Parsing\HTML\HTMLStreamElements\Element;
use Kibo\Phast\Parsing\HTML\HTMLStreamElements\Tag;

class PCRETokenizer {

    private $mainPattern = "~
        # Allow duplicate names for subpatterns
        (?J)

        (
            @@COMMENT |
            @@SCRIPT |
            @@STYLE |
            @@CLOSING_TAG |
            @@TAG
        )
    ~Xxs";

    private $attributePattern = "~
        @attr
    ~Xxs";

    private $subroutines = [
        'COMMENT' => "
            <!--.*?-->
        ",
        'SCRIPT' => "
            < (?'tag_name' script ) @@attrs @tag_end
            (?'body' .*? )
            (?'closing_tag' </script[^a-z>]*+> )
        ",
        'STYLE' => "
            < (?'tag_name' style ) @@attrs @tag_end
            (?'body' .*? )
            (?'closing_tag' </style[^a-z>]*+> )
        ",
        'TAG' => "
            < @@tag_name @@attrs @tag_end
        ",
        'tag_name' => "
            [^\s>]++
        ",
        'attrs' => "
            ( @attr )*+
        ",
        'attr' => "
            \s++
            @@attr_name
            (?: \s*+ = \s*+ @attr_value )?
        ",
        'attr_name' => "
            [^\s>][^\s>=]*+
        ",
        'attr_value' => "
            \"(?'attr_value'[^\"]*+)\"
        ",
        'tag_end' => "
            \s*+ >
        ",
        'CLOSING_TAG' => "
            </ @@tag_name [^>]*+ >
        "
    ];

    public function __construct() {
        $this->mainPattern = $this->compilePattern($this->mainPattern, $this->subroutines);
        $this->attributePattern = $this->compilePattern($this->attributePattern, $this->subroutines);
    }

    public function tokenize($data) {
        $offset = 0;

        while (preg_match($this->mainPattern, $data, $match, PREG_OFFSET_CAPTURE, $offset)) {
            if ($match[0][1] > $offset) {
                $element = new Element();
                $element->setOriginalString(substr($data, $offset, $match[0][1] - $offset));
                yield $element;
            }
            if (!empty($match['COMMENT'][0])) {
                $element = new Element();
            } elseif (!empty($match['TAG'][0])
                      || !empty($match['SCRIPT'][0])
                      || !empty($match['STYLE'][0])
            ) {
                $attributes = $this->parseAttributes($match['attrs'][0]);
                $element = new Tag($match['tag_name'][0], $attributes);
                if (isset($match['body'][1]) && $match['body'][1] != -1) {
                    $element->setTextContent($match['body'][0]);
                    $element = $element->withClosingTag($match['closing_tag'][0]);
                }
            } elseif (!empty($match['CLOSING_TAG'][0])) {
                $element = new ClosingTag($match['tag_name'][0]);
            } else {
                throw new RuntimeException("Unhandled match:\n" . json_encode($match, JSON_PRETTY_PRINT));
            }
            $element->setOriginalString($match[0][0]);
            yield $element;
            $offset = $match[0][1] + strlen($match[0][0]);
        }

        if ($offset < strlen($data) - 1) {
            $element = new Element();
            $element->setOriginalString(substr($data, $offset));
            yield $element;
        }
    }

    private function parseAttributes($str) {
        $matches = $this->repeatMatch($this->attributePattern, $str);
        $attributes = [];

        foreach ($matches as $match) {
            // TODO: Move this to Tag. Do not decode/recode unless a specific attribute is changed.
            $attributes[$match['attr_name']] =
                isset($match['attr_value']) ? html_entity_decode($match['attr_value'], ENT_QUOTES, 'UTF-8') : '';
        }

        return $attributes;
    }

    private function repeatMatch($pattern, $subject) {
        $offset = 0;

        while (preg_match($pattern, $subject, $match, PREG_OFFSET_CAPTURE, $offset)) {
            $offset = $match[0][1] + strlen($match[0][0]);
            $match = array_map(function ($group) { return $group[0]; }, $match);
            yield $match;
        }

        if ($offset < strlen($subject) - 1) {
            throw new RuntimeException("Unmatched part of subject: " . substr($subject, $offset));
        }
    }

    /**
     * Replace subroutines in patterns
     */
    private function compilePattern($pattern, array $subroutines) {
        return preg_replace_callback('/@(@?)(\w+)/', function ($match) use ($subroutines) {
            $capture = !empty($match[1]);
            $ref = $match[2];

            if (!isset($subroutines[$ref])) {
                throw new RuntimeException(
                    "Unknown pattern '$ref' used, or circular reference");
            }

            $subroutine = $subroutines[$ref];
            unset($subroutines[$ref]);

            $replace = $this->compilePattern($subroutine, $subroutines);

            if ($capture) {
                $replace = "(?'$ref'$replace)";
            } else {
                $replace = "(?:$replace)";
            }
<commit_msg>PCRETokenizer: Speed up repeatMatch<commit_after><?php

namespace Kibo\Phast\Parsing\HTML;

use Kibo\Phast\Exceptions\RuntimeException;
use Kibo\Phast\Parsing\HTML\HTMLStreamElements\ClosingTag;
use Kibo\Phast\Parsing\HTML\HTMLStreamElements\Element;
use Kibo\Phast\Parsing\HTML\HTMLStreamElements\Tag;

class PCRETokenizer {

    private $mainPattern = "~
        # Allow duplicate names for subpatterns
        (?J)

        (
            @@COMMENT |
            @@SCRIPT |
            @@STYLE |
            @@CLOSING_TAG |
            @@TAG
        )
    ~Xxs";

    private $attributePattern = "~
        @attr
    ~Xxs";

    private $subroutines = [
        'COMMENT' => "
            <!--.*?-->
        ",
        'SCRIPT' => "
            < (?'tag_name' script ) @@attrs @tag_end
            (?'body' .*? )
            (?'closing_tag' </script[^a-z>]*+> )
        ",
        'STYLE' => "
            < (?'tag_name' style ) @@attrs @tag_end
            (?'body' .*? )
            (?'closing_tag' </style[^a-z>]*+> )
        ",
        'TAG' => "
            < @@tag_name @@attrs @tag_end
        ",
        'tag_name' => "
            [^\s>]++
        ",
        'attrs' => "
            ( @attr )*+
        ",
        'attr' => "
            \s++
            @@attr_name
            (?: \s*+ = \s*+ @attr_value )?
        ",
        'attr_name' => "
            [^\s>][^\s>=]*+
        ",
        'attr_value' => "
            \"(?'attr_value'[^\"]*+)\"
        ",
        'tag_end' => "
            \s*+ >
        ",
        'CLOSING_TAG' => "
            </ @@tag_name [^>]*+ >
        "
    ];

    public function __construct() {
        $this->mainPattern = $this->compilePattern($this->mainPattern, $this->subroutines);
        $this->attributePattern = $this->compilePattern($this->attributePattern, $this->subroutines);
    }

    public function tokenize($data) {
        $offset = 0;

        while (preg_match($this->mainPattern, $data, $match, PREG_OFFSET_CAPTURE, $offset)) {
            if ($match[0][1] > $offset) {
                $element = new Element();
                $element->setOriginalString(substr($data, $offset, $match[0][1] - $offset));
                yield $element;
            }
            if (!empty($match['COMMENT'][0])) {
                $element = new Element();
            } elseif (!empty($match['TAG'][0])
                      || !empty($match['SCRIPT'][0])
                      || !empty($match['STYLE'][0])
            ) {
                $attributes = $this->parseAttributes($match['attrs'][0]);
                $element = new Tag($match['tag_name'][0], $attributes);
                if (isset($match['body'][1]) && $match['body'][1] != -1) {
                    $element->setTextContent($match['body'][0]);
                    $element = $element->withClosingTag($match['closing_tag'][0]);
                }
            } elseif (!empty($match['CLOSING_TAG'][0])) {
                $element = new ClosingTag($match['tag_name'][0]);
            } else {
                throw new RuntimeException("Unhandled match:\n" . json_encode($match, JSON_PRETTY_PRINT));
            }
            $element->setOriginalString($match[0][0]);
            yield $element;
            $offset = $match[0][1] + strlen($match[0][0]);
        }

        if ($offset < strlen($data) - 1) {
            $element = new Element();
            $element->setOriginalString(substr($data, $offset));
            yield $element;
        }
    }

    private function parseAttributes($str) {
        $matches = $this->repeatMatch($this->attributePattern, $str);
        $attributes = [];

        foreach ($matches as $match) {
            // TODO: Move this to Tag. Do not decode/recode unless a specific attribute is changed.
            $attributes[$match['attr_name'][0]] =
                isset($match['attr_value'][0]) ? html_entity_decode($match['attr_value'][0], ENT_QUOTES, 'UTF-8') : '';
        }

        return $attributes;
    }

    private function repeatMatch($pattern, $subject) {
        $offset = 0;

        while (preg_match($pattern, $subject, $match, PREG_OFFSET_CAPTURE, $offset)) {
            yield $match;
            $offset = $match[0][1] + strlen($match[0][0]);
        }

        if ($offset < strlen($subject) - 1) {
            throw new RuntimeException("Unmatched part of subject: " . substr($subject, $offset));
        }
    }

    /**
     * Replace subroutines in patterns
     */
    private function compilePattern($pattern, array $subroutines) {
        return preg_replace_callback('/@(@?)(\w+)/', function ($match) use ($subroutines) {
            $capture = !empty($match[1]);
            $ref = $match[2];

            if (!isset($subroutines[$ref])) {
                throw new RuntimeException(
                    "Unknown pattern '$ref' used, or circular reference");
            }

            $subroutine = $subroutines[$ref];
            unset($subroutines[$ref]);

            $replace = $this->compilePattern($subroutine, $subroutines);

            if ($capture) {
                $replace = "(?'$ref'$replace)";
            } else {
                $replace = "(?:$replace)";
            }

{'fraction_non_alphanumeric': 0.1741011343972313, 'fraction_numerical': 0.006921745818111902, 'mean_word_length': 1.7369113391212838, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5305748', 'n_tokens_mistral': 3176, 'n_tokens_neox': 3034, 'n_words': 529}