<commit_before><?php namespace Laurentvw\Scrapher;

use Laurentvw\Scrapher\Selectors\Selector;

class Scrapher {

    /**
     * The scraper's contents
     *
     * @var array
     */
    protected $contents = array();

    /**
     * The crawler's matcher.
     *
     * @var \Laurentvw\Scrapher\Matcher
     */
    protected $matcher;

    /**
     * The number of matches to take
     *
     * @var int
     */
    protected $take;

    /**
     * The number of matches to skip
     *
     * @var int
     */
    protected $skip = 0;

    /**
     * The order of the matches
     *
     * @var array
     */
    protected $orderBy;

    /**
     * Create a new Scrapher instance.
     *
     * @param string|array $contents
     */
    public function __construct($contents)
    {
        $this->setContents($contents);
    }

    /**
     * Set the type of selector to use
     *
     * @param Selector $selector
     * @return Scrapher
     */
    public function with(Selector $selector)
    {
        $this->matcher = new Matcher($selector);

        return $this;
    }

    /**
     * Filter the resulting matches
     *
     * @param  \Closure  $filter
     * @return Scrapher
     */
    public function filter($filter)
    {
        $this->getMatcher()->setFilter($filter);

        return $this;
    }

    /**
     * Take n-number of matches
     *
     * @param int $n
     * @return Scrapher
     */
    public function take($n)
    {
        $this->take = $n;

        return $this;
    }
    /**
     * Skip n-number of matches
     *
     * @param int $n
     * @return Scrapher
     */
    public function skip($n)
    {
        $this->skip = $n;

        return $this;
    }
    /**
     * Order the matches
     *
     * @param string $name
     * @param string $order
     * @param string $projection
     * @return Scrapher
     */
    public function orderBy($name, $order = 'asc', $projection = null)
    {
        $order = strtolower($order) == 'desc' ? SORT_DESC : SORT_ASC;
        $this->orderBy[] = array($name, $order, $projection);

        return $this;
    }

    /**
     * Get all the matches
     *
     * @return array
     */
    public function get()
    {
        return $this->scrape();
    }

    /**
     * Get the first match
     *
     * @return array
     */
    public function first()
    {
        $results = $this->scrape();

        return current($results);
    }

    /**
     * Get the last match
     *
     * @return array
     */
    public function last()
    {
        $results = $this->scrape();

        return end($results);
    }

    /**
     * Count the number of matches
     *
     * @return array
     */
    public function count()
    {
        $results = $this->scrape();

        return count($results);
    }

    /**
     * Get detailed logs of the scraping
     *
     * @return array
     */
    public function getLogs()
    {
        return $this->getMatcher()->getLogs();
    }

    /**
     * Set the contents to be scraped
     *
     * Can either be the actual content (HTML, ...) or the URL
     *
     * You may pass multiple content at once by using an array
     *
     * @param string|array $contents
     */
    protected function setContents($contents)
    {
        if (is_array($contents))
        {
            foreach ($contents as $content)
            {
                $this->setContent($content);
            }
        }

        $this->setContent($contents);
    }

    /**
     * Set a content to be scraped
     *
     * Can either be the actual content (HTML, ...) or the URL
     *
     * @param string $content
     */
    protected function setContent($content)
    {
        if (substr($content, 0, 4) == 'http')
        {
            $page = new Page($content);
            $this->contents[] = $page->getHTML();
        }
        else
        {
            $this->contents[] = $content;
        }
    }

    /**
     * The matcher
     *
     * @return Matcher
     */
    protected function getMatcher()
    {
        return $this->matcher;
    }<commit_msg>Fix bug with multiple contents<commit_after><?php namespace Laurentvw\Scrapher;

use Laurentvw\Scrapher\Selectors\Selector;

class Scrapher {

    /**
     * The scraper's contents
     *
     * @var array
     */
    protected $contents = array();

    /**
     * The crawler's matcher.
     *
     * @var \Laurentvw\Scrapher\Matcher
     */
    protected $matcher;

    /**
     * The number of matches to take
     *
     * @var int
     */
    protected $take;

    /**
     * The number of matches to skip
     *
     * @var int
     */
    protected $skip = 0;

    /**
     * The order of the matches
     *
     * @var array
     */
    protected $orderBy;

    /**
     * Create a new Scrapher instance.
     *
     * @param string|array $contents
     */
    public function __construct($contents)
    {
        $this->setContents($contents);
    }

    /**
     * Set the type of selector to use
     *
     * @param Selector $selector
     * @return Scrapher
     */
    public function with(Selector $selector)
    {
        $this->matcher = new Matcher($selector);

        return $this;
    }

    /**
     * Filter the resulting matches
     *
     * @param  \Closure  $filter
     * @return Scrapher
     */
    public function filter($filter)
    {
        $this->getMatcher()->setFilter($filter);

        return $this;
    }

    /**
     * Take n-number of matches
     *
     * @param int $n
     * @return Scrapher
     */
    public function take($n)
    {
        $this->take = $n;

        return $this;
    }
    /**
     * Skip n-number of matches
     *
     * @param int $n
     * @return Scrapher
     */
    public function skip($n)
    {
        $this->skip = $n;

        return $this;
    }
    /**
     * Order the matches
     *
     * @param string $name
     * @param string $order
     * @param string $projection
     * @return Scrapher
     */
    public function orderBy($name, $order = 'asc', $projection = null)
    {
        $order = strtolower($order) == 'desc' ? SORT_DESC : SORT_ASC;
        $this->orderBy[] = array($name, $order, $projection);

        return $this;
    }

    /**
     * Get all the matches
     *
     * @return array
     */
    public function get()
    {
        return $this->scrape();
    }

    /**
     * Get the first match
     *
     * @return array
     */
    public function first()
    {
        $results = $this->scrape();

        return current($results);
    }

    /**
     * Get the last match
     *
     * @return array
     */
    public function last()
    {
        $results = $this->scrape();

        return end($results);
    }

    /**
     * Count the number of matches
     *
     * @return array
     */
    public function count()
    {
        $results = $this->scrape();

        return count($results);
    }

    /**
     * Get detailed logs of the scraping
     *
     * @return array
     */
    public function getLogs()
    {
        return $this->getMatcher()->getLogs();
    }

    /**
     * Set the contents to be scraped
     *
     * Can either be the actual content (HTML, ...) or the URL
     *
     * You may pass multiple content at once by using an array
     *
     * @param string|array $contents
     */
    protected function setContents($contents)
    {
        if (is_array($contents))
        {
            foreach ($contents as $content)
            {
                $this->setContent($content);
            }
        }
        else
        {
            $this->setContent($contents);
        }
    }

    /**
     * Set a content to be scraped
     *
     * Can either be the actual content (HTML, ...) or the URL
     *
     * @param string $content
     */
    protected function setContent($content)
    {
        if (substr($content, 0, 4) == 'http')
        {
            $page = new Page($content);
            $this->contents[] = $page->getHTML();
        }
        else
        {
            $this->contents[] = $content;
        }
    }

    /**
     * The matcher
     *
     * @return Matcher
     */
    protected function getMatcher()
    {
        return $this->matcher;
    }
{'fraction_non_alphanumeric': 0.12745696496154316, 'fraction_numerical': 0.000732511292882432, 'mean_word_length': 1.4892130051656032, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1669553', 'n_tokens_mistral': 2513, 'n_tokens_neox': 2434, 'n_words': 631}