<commit_before><?php

namespace Prontotype\Service\Scraper;

use Symfony\Component\CssSelector\CssSelector;

class Scrap implements ScrapInterface {

	protected $full_html;
	protected $html;
	protected $uri;
	protected $dom;
	protected $xpath;

    public function __construct( $html, $uri )
    {
        $this->html = $html;
        $this->uri = $uri;

        $this->full_html = strpos($html, '<html') > -1;
    }

    protected function domBootstrap () {
        $this->dom = new \DOMDocument('1.0', 'utf8');
        $this->dom->validateOnParse = false;

        $current = libxml_use_internal_errors(true);
        $this->dom->loadHTML($this->html);
        libxml_use_internal_errors($current);

        $this->xpath = new \DOMXpath($this->dom);

        // change href and src attribute values
        foreach (array('href', 'src') as $attr) {
            foreach ($this->xpath->query("//*[@" . $attr . "]") as $element) {
                $url = $this->absolute_url($this->uri, $element->getAttribute($attr));
                $element->setAttribute($attr, $url);
            }
        }
    }

    public function filter($filter)
    {
        $this->domBootstrap();

        $collection = new ScrapCollection();

        foreach ($this->xpath->query(CssSelector::toXPath($filter)) as $element) {
            $collection->add(new Scrap($this->dom->saveHtml($element), $this->uri));
        }

        return $collection;
    }

    public function insert($filter, $html)
    {
        $this->domBootstrap();

        $html = trim ( (string) $html);
        if (empty($html) ) {
            return $this;
        }

        $tempDom = new \DOMDocument('1.0', 'utf8');
        $tempDom->loadHTML($html);
        $import = $tempDom->getElementsByTagName('body')->item(0);

        foreach ($this->xpath->query(CssSelector::toXPath($filter)) as $element) {
            $firstChild = $element->firstChild;

            foreach ($import->childNodes as $child) {
                $importedNode = $this->dom->importNode($child, true);
                $element->insertBefore($importedNode, $firstChild);
            }
        }

        return $this;
    }

    /*<commit_msg>every action on scrap returns a new scrap or scrapcollection<commit_after><?php

namespace Prontotype\Service\Scraper;

use Symfony\Component\CssSelector\CssSelector;

class Scrap implements ScrapInterface {

	protected $full_html;
	protected $html;
	protected $uri;
	protected $dom;
	protected $xpath;

    public function __construct($html, $uri = '')
    {
        $this->html = $html;
        $this->uri = $uri;

        $this->full_html = strpos($html, '<html') > -1;
    }

    protected function domBootstrap () {
        $this->dom = new \DOMDocument('1.0', 'utf8');
        $this->dom->validateOnParse = false;

        $current = libxml_use_internal_errors(true);
        $this->dom->loadHTML($this->html);
        libxml_use_internal_errors($current);

        $this->xpath = new \DOMXpath($this->dom);

        if ($this->uri) {
            // change href and src attribute values
            foreach (array('href', 'src') as $attr) {
                foreach ($this->xpath->query("//*[@" . $attr . "]") as $element) {
                    $url = $this->absolute_url($this->uri, $element->getAttribute($attr));
                    $element->setAttribute($attr, $url);
                }
            }
        }
    }

    public function filter($filter)
    {
        $this->domBootstrap();

        $collection = new ScrapCollection();

        foreach ($this->xpath->query(CssSelector::toXPath($filter)) as $element) {
            $collection->add(new Scrap($this->dom->saveHtml($element)));
        }

        return $collection;
    }

    public function insert($filter, $html)
    {
        $this->domBootstrap();

        $html = trim ( (string) $html);
        if (empty($html) ) {
            return $this;
        }

        $tempDom = new \DOMDocument('1.0', 'utf8');
        $tempDom->loadHTML($html);
        $import = $tempDom->getElementsByTagName('body')->item(0);

        foreach ($this->xpath->query(CssSelector::toXPath($filter)) as $element) {
            $firstChild = $element->firstChild;

            foreach ($import->childNodes as $child) {
                $importedNode = $this->dom->importNode($child, true);
                $element->insertBefore($importedNode, $firstChild);
            }
        }

        return new Scrap($this->dom->saveHtml());
    }

    /*
{'fraction_non_alphanumeric': 0.16041108132260948, 'fraction_numerical': 0.0035746201966041107, 'mean_word_length': 2.2583697234352256, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 77, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2001859', 'n_tokens_mistral': 1358, 'n_tokens_neox': 1317, 'n_words': 254}