<commit_before><?php

namespace Mihaeu\Tarantula;

use Mihaeu\Tarantula\Action\ActionInterface;
use Mihaeu\Tarantula\Filter\FilterInterface;

use Symfony\Component\DomCrawler\Crawler as DOMCrawler;

/**
 * Guzzle based Crawler.
 *
 * @author <NAME> <<EMAIL>>
 */
class Crawler
{
    /**
     * @var HttpClient
     */
    private $client;

    /**
     * @var Array
     */
    private $allLinks = array();

    /**
     * @var Array
     */
    private $actions = array();

    /**
     * @var Array
     */
    private $filters = array();

    /**
     * Constructor.
     *
     * @param HttpClient $client
     */
    public function __construct(HttpClient $client)
    {
        $this->client = $client;
    }

    /**
     * This is the main action that will get called recursively depending on `$depth`.
     * 
     * @param  integer $depth
     * @param  String  $url
     * 
     * @return Array
     */
    public function go($depth = -1, $url = '')
    {
        if ($url === '') {
            $url = $this->client->getStartUrl();
        }

        if ($url !== $this->client->getStartUrl() && !$this->urlPassesFilters($url)) {
            return array();
        }

        // download from the url
        $data = $this->client->downloadContent($url);
        if (empty($data)) {
            return array();
        }
        $this->processActions($url, $data);

        // when we reach max. depth we don't need to go deeper and download more
        if ($depth-- !== 0) {
            // parse sub links
            $links = $this->findAllLinks($data);
            $this->allLinks = array_merge($this->allLinks, $links);

            // recursive calls provide depth
            foreach ($links as $hash => $link) {
                $this->allLinks = array_merge($this->allLinks, $this->go($depth, $link));
            }
        }

        return $this->allLinks;
    }

    /**
     * Runs all filters against the url.
     *
     * @param String $url
     *
     * @return bool<commit_msg>Applies url filters one step earlier.<commit_after><?php

namespace Mihaeu\Tarantula;

use Mihaeu\Tarantula\Action\ActionInterface;
use Mihaeu\Tarantula\Filter\FilterInterface;

use Symfony\Component\DomCrawler\Crawler as DOMCrawler;

/**
 * Guzzle based Crawler.
 *
 * @author <NAME> <<EMAIL>>
 */
class Crawler
{
    /**
     * @var HttpClient
     */
    private $client;

    /**
     * @var Array
     */
    private $allLinks = array();

    /**
     * @var Array
     */
    private $actions = array();

    /**
     * @var Array
     */
    private $filters = array();

    /**
     * Constructor.
     *
     * @param HttpClient $client
     */
    public function __construct(HttpClient $client)
    {
        $this->client = $client;
    }

    /**
     * This is the main action that will get called recursively depending on `$depth`.
     * 
     * @param  integer $depth
     * @param  String  $url
     * 
     * @return Array
     */
    public function go($depth = -1, $url = '')
    {
        if ($url === '') {
            $url = $this->client->getStartUrl();
        }

        // apply filters
        if ($url !== $this->client->getStartUrl() && !$this->urlPassesFilters($url)) {
            return array();
        }

        // download from the url
        $data = $this->client->downloadContent($url);
        if (empty($data)) {
            return array();
        }
        $this->processActions($url, $data);

        // add current url to links
        $currentHash = $this->client->createHashFromUrl($url);
        if (!isset($this->allLinks[$currentHash])) {
            $this->allLinks[$currentHash] = $this->client->convertToAbsoluteUrl($url);
        }

        // when we reach max. depth we don't need to go deeper and download more
        if ($depth-- !== 0) {
            // parse sub links
            $links = $this->filterUrls($this->findAllLinks($data));
            $this->allLinks = array_merge($this->allLinks, $links);

            // recursive calls provide depth
            foreach ($links as $hash => $link) {
                $this->allLinks = array_merge($this->allLinks, $this->go($depth, $link));
            }
        }

        return $this->allLinks;
    }

    public function filterUrls(Array $urls)
    {
        $filteredUrls = array();
        foreach ($urls as $hash => $url) {
            if ($this->urlPassesFilters($url)) {
                $filteredUrls[$hash] = $url;
            }
        }
        return $filteredUrls;
    }

    /**
     * Runs all filters against the url.
     *
     * @param String $url
     *
     * @return bool
{'fraction_non_alphanumeric': 0.14790717848622859, 'fraction_numerical': 0.0008674907829104315, 'mean_word_length': 1.7951515151515152, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 52, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3082026', 'n_tokens_mistral': 1401, 'n_tokens_neox': 1359, 'n_words': 328}