<commit_before><?php
/**
 *  base include file for SimpleTest
 *  @package    SimpleTest
 *  @subpackage WebTester
 *  @version    $Id: php_parser.php 1911 2009-07-29 16:38:04Z lastcraft $
 */

/**
 *    Builds the page object.
 *    @package SimpleTest
 *    @subpackage WebTester
 */
class SimpleTidyPageBuilder {
    private $page;
    private $forms = array();
    private $labels = array();

    public function __destruct() {
        unset($this->page);
        unset($this->forms);
        unset($this->labels);
    }

    /**
     *    Reads the raw content the page.
     *    @param $response SimpleHttpResponse  Fetched response.
     *    @return SimplePage                   Newly parsed page.
     *    @access public
     */
    function parse($response) {
        $this->page = new SimplePage($response);
        $tidied = tidy_parse_string($response->getContent(), array('output-html' => true), 'latin1');
        if ($tidied->errorBuffer) {
            foreach(explode("\n", $tidied->errorBuffer) as $notice) {
                //user_error($notice, E_USER_NOTICE);
            }
        }
        $this->walkTree($tidied->html());
        $this->page->setForms($this->attachLabels($this->forms, $this->labels));
        return $this->page;
    }

    /**
     * Visits the given node and all children
     */
    private function walkTree($node) {
        if ($node->name == 'a') {
            $this->page->addLink($this->tags()->createTag($node->name, (array)$node->attribute)
                                        ->addContent($this->innerHtml($node)));
        } elseif ($node->name == 'base') {
            $this->page->setBase($node->attribute['href']);
        } elseif ($node->name == 'title') {
            $this->page->setTitle($this->tags()->createTag($node->name, (array)$node->attribute)
                                         ->addContent($this->innerHtml($node)));
        } elseif ($node->name == 'frameset') {
            $this->page->setFrames($this->collectFrames($node));
        } elseif ($node->name == 'form') {
            $this->forms[] = $this->walkForm($node, $this->createEmptyForm($node));
        } elseif ($node->name == 'label') {
            $this->labels[] = $this->tags()->createTag($node->name, (array)$node->attribute)
                                           ->addContent($this->innerHtml($node));
        } else {
            $this->walkChildren($node);
        }
    }

    private function createEmptyForm($node) {
        return new SimpleForm($this->tags()->createTag($node->name, (array)$node->attribute), $this->page);
    }

    private function walkForm($node, $form) {
        if ($node->name == 'a') {
            $this->page->addLink($this->tags()->createTag($node->name, (array)$node->attribute)
                                        ->addContent($this->innerHtml($node)));
        } elseif ($node->name == 'input') {
            $form->addWidget($this->tags()->createTag($node->name, (array)$node->attribute));
        } elseif ($node->name == 'button' || $node->name == 'textarea') {
            $form->addWidget($this->tags()->createTag($node->name, (array)$node->attribute)
                                          ->addContent($this->innerHtml($node)));
        } elseif ($node->name == 'label') {
            $this->labels[] = $this->tags()->createTag($node->name, (array)$node->attribute)
                                           ->addContent($this->innerHtml($node));
        }
        if ($node->hasChildren()) {
            foreach ($node->child as $child) {
                $this->walkForm($child, $form);
            }
        }
        return $form;
    }

    private function collectFrames($node) {<commit_msg>Start working on select/option parsing<commit_after><?php
/**
 *  base include file for SimpleTest
 *  @package    SimpleTest
 *  @subpackage WebTester
 *  @version    $Id: php_parser.php 1911 2009-07-29 16:38:04Z lastcraft $
 */

/**
 *    Builds the page object.
 *    @package SimpleTest
 *    @subpackage WebTester
 */
class SimpleTidyPageBuilder {
    private $page;
    private $forms = array();
    private $labels = array();

    public function __destruct() {
        unset($this->page);
        unset($this->forms);
        unset($this->labels);
    }

    /**
     *    Reads the raw content the page.
     *    @param $response SimpleHttpResponse  Fetched response.
     *    @return SimplePage                   Newly parsed page.
     *    @access public
     */
    function parse($response) {
        $this->page = new SimplePage($response);
        $tidied = tidy_parse_string($response->getContent(), array('output-html' => true), 'latin1');
        if ($tidied->errorBuffer) {
            foreach(explode("\n", $tidied->errorBuffer) as $notice) {
                //user_error($notice, E_USER_NOTICE);
            }
        }
        $this->walkTree($tidied->html());
        $this->page->setForms($this->attachLabels($this->forms, $this->labels));
        return $this->page;
    }

    /**
     * Visits the given node and all children
     */
    private function walkTree($node) {
        if ($node->name == 'a') {
            $this->page->addLink($this->tags()->createTag($node->name, (array)$node->attribute)
                                        ->addContent($this->innerHtml($node)));
        } elseif ($node->name == 'base') {
            $this->page->setBase($node->attribute['href']);
        } elseif ($node->name == 'title') {
            $this->page->setTitle($this->tags()->createTag($node->name, (array)$node->attribute)
                                         ->addContent($this->innerHtml($node)));
        } elseif ($node->name == 'frameset') {
            $this->page->setFrames($this->collectFrames($node));
        } elseif ($node->name == 'form') {
            $this->forms[] = $this->walkForm($node, $this->createEmptyForm($node));
        } elseif ($node->name == 'label') {
            $this->labels[] = $this->tags()->createTag($node->name, (array)$node->attribute)
                                           ->addContent($this->innerHtml($node));
        } else {
            $this->walkChildren($node);
        }
    }

    private function createEmptyForm($node) {
        return new SimpleForm($this->tags()->createTag($node->name, (array)$node->attribute), $this->page);
    }

    private function walkForm($node, $form) {
        if ($node->name == 'a') {
            $this->page->addLink($this->tags()->createTag($node->name, (array)$node->attribute)
                                        ->addContent($this->innerHtml($node)));
        } elseif ($node->name == 'input') {
            $form->addWidget($this->tags()->createTag($node->name, (array)$node->attribute));
        } elseif ($node->name == 'button' || $node->name == 'textarea') {
            $form->addWidget($this->tags()->createTag($node->name, (array)$node->attribute)
                                          ->addContent($this->innerHtml($node)));
        } elseif ($node->name == 'label') {
            $this->labels[] = $this->tags()->createTag($node->name, (array)$node->attribute)
                                           ->addContent($this->innerHtml($node));
        } elseif ($node->name == 'select') {
            $select = $this->tags()->createTag($node->name, (array)$node->attribute);
            $form->addWidget($this->collectSelectOptions($select, $node));
        }
        if ($node->hasChildren()) {
            foreach ($node->child as $child) {
                $this->walkForm($child, $form);
            }
        }
        return $form;
    }

    private function collectSelectOptions($select, $node) {
        if ($node->name == 'option') {
            $select->addTag($this->tags()->createTag($node->name, (array)$node->attribute)
                                         ->addContent($this->innerHtml($node)));
        }
        if ($node->hasChildren()) {
            foreach ($node->child as $child) {
                $this->collectSelectOptions($select, $child);
            }
        }
        return $select;
    }

    private function collectFrames($node) {
{'fraction_non_alphanumeric': 0.18238916256157636, 'fraction_numerical': 0.004679802955665025, 'mean_word_length': 2.0427126264518547, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 211, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6490110', 'n_tokens_mistral': 2242, 'n_tokens_neox': 2203, 'n_words': 388}