<commit_before><?php

/*
 * Tocifier is intentionally decoupled from SilverStripe to be able to
 * test it without needing to put all the test infrastructure up.
 */
class Tocifier
{
    // Prefix to prepend to every URL fragment
    public static $prefix = 'TOC-';

    // The original HTML
    private $_raw_html = '';

    // $_raw_html augmented with anchor ids for proper navigation
    private $_html = '';

    // The most recently generated TOC tree.
    private $_tree;

    // Array of references to the potential parents
    private $_dangling = array();


    private function &_getParent($level)
    {
        while (--$level >= 0) {
            if (isset($this->_dangling[$level])) {
                return $this->_dangling[$level];
            }
        }
        // This should never be reached
        assert(false);
    }

    private function _getPlainText(DOMElement $tag)
    {
        // Work on a copy
        $clone = $tag->cloneNode(true);

        // Strip unneded tags (<small>)
        while (($tag = $clone->getElementsByTagName('small')) && $tag->length) {
            $tag->item(0)->parentNode->removeChild($tag->item(0));
        }

        return $clone->textContent;
    }

    private function &_newNode($id, $text, $level)
    {
        $node = array(
            'id'    => $id,
            'title' => $text
        );

        // Clear the trailing dangling parents after level, if any
        end($this->_dangling);
        $last = key($this->_dangling);
        for ($n = $level+1; $n <= $last; ++$n) {
            unset($this->_dangling[$n]);
        }

        // Consider this node a potential dangling parent
        $this->_dangling[$level] =& $node;

        return $node;
    }

    private function _processDocument($doc)
    {
        $this->_tree =& $this->_newNode(self::$prefix, '', 0);
        $n = 1;

        $xpath = new DOMXPath($doc);
        $query = '//h1[not(@data-hide-from-toc)]|//h2[not(@data-hide-from-toc)]|//h3[not(@data-hide-from-toc)]|//h4[not(@data-hide-from-toc)]|//h5[not(@data-hide-from-toc)]|//h6[not(@data-hide-from-toc)]';

        foreach ($xpath->query($query) as $h) {
            $text = $this->_getPlainText($h);
            $level = (int) substr($h->tagName, 1);
            $id = self::$prefix . $n;
            ++$n;

            // Build the tree
            $parent =& $this->_getParent($level);
            $node =& $this->_newNode($id, $text, $level);
            isset($parent['children']) or $parent['children'] = array();
            $parent['children'][] =& $node;

            // Prepend the anchor
            $anchor = $doc->createElement('a');
            $anchor->setAttribute('id', $id);
            $anchor->setAttribute('class', 'anchor');
            $h->parentNode->insertBefore($anchor, $h);
        }

        $body = $doc->getElementsByTagName('body')->item(0);
        $this->_html = str_replace(array("<body>\n", '</body>'), '',
                                   $doc->saveHTML($body));<commit_msg>Remove some redundancy from XPath expression<commit_after><?php

/*
 * Tocifier is intentionally decoupled from SilverStripe to be able to
 * test it without needing to put all the test infrastructure up.
 */
class Tocifier
{
    // Prefix to prepend to every URL fragment
    public static $prefix = 'TOC-';

    // The original HTML
    private $_raw_html = '';

    // $_raw_html augmented with anchor ids for proper navigation
    private $_html = '';

    // The most recently generated TOC tree.
    private $_tree;

    // Array of references to the potential parents
    private $_dangling = array();


    private function &_getParent($level)
    {
        while (--$level >= 0) {
            if (isset($this->_dangling[$level])) {
                return $this->_dangling[$level];
            }
        }
        // This should never be reached
        assert(false);
    }

    private function _getPlainText(DOMElement $tag)
    {
        // Work on a copy
        $clone = $tag->cloneNode(true);

        // Strip unneded tags (<small>)
        while (($tag = $clone->getElementsByTagName('small')) && $tag->length) {
            $tag->item(0)->parentNode->removeChild($tag->item(0));
        }

        return $clone->textContent;
    }

    private function &_newNode($id, $text, $level)
    {
        $node = array(
            'id'    => $id,
            'title' => $text
        );

        // Clear the trailing dangling parents after level, if any
        end($this->_dangling);
        $last = key($this->_dangling);
        for ($n = $level+1; $n <= $last; ++$n) {
            unset($this->_dangling[$n]);
        }

        // Consider this node a potential dangling parent
        $this->_dangling[$level] =& $node;

        return $node;
    }

    private function _processDocument($doc)
    {
        $this->_tree =& $this->_newNode(self::$prefix, '', 0);
        $n = 1;

        $xpath = new DOMXPath($doc);
        $query = '//*[translate(name(), "123456", "......") = "h."][not(@data-hide-from-toc)]';

        foreach ($xpath->query($query) as $h) {
            $text = $this->_getPlainText($h);
            $level = (int) substr($h->tagName, 1);
            $id = self::$prefix . $n;
            ++$n;

            // Build the tree
            $parent =& $this->_getParent($level);
            $node =& $this->_newNode($id, $text, $level);
            isset($parent['children']) or $parent['children'] = array();
            $parent['children'][] =& $node;

            // Prepend the anchor
            $anchor = $doc->createElement('a');
            $anchor->setAttribute('id', $id);
            $anchor->setAttribute('class', 'anchor');
            $h->parentNode->insertBefore($anchor, $h);
        }

        $body = $doc->getElementsByTagName('body')->item(0);
        $this->_html = str_replace(array("<body>\n", '</body>'), '',
                                   $doc->saveHTML($body));
{'fraction_non_alphanumeric': 0.16888589246586888, 'fraction_numerical': 0.004719366256531266, 'mean_word_length': 2.0030364372469633, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '63176', 'n_tokens_mistral': 1811, 'n_tokens_neox': 1722, 'n_words': 416}