<commit_before><?php
/**
 * @todo adds a description (license text, description of this class / file, etc)
 */
namespace TYPO3Analysis\Consumer\Crawler;

use TYPO3Analysis\Consumer\ConsumerAbstract;
use Symfony\Component\DomCrawler\Crawler;

class Gitweb extends ConsumerAbstract {

    /**
     * HTTP Client
     *
     * @var \Buzz\Browser
     */
    protected $browser = null;

    /**
     * Gets a description of the consumer
     *
     * @return string
     */
    public function getDescription() {
        return 'Crawls a Gitweb-Index page for Git-repositories';
    }

    /**
     * Initialize the consumer.
     * Sets the queue and routing key
     *
     * @return void
     */
    public function initialize() {
        $this->setQueue('crawler.gitweb');
        $this->setRouting('crawler.gitweb');

        $curlClient = new \Buzz\Client\Curl();
        $curlClient->setVerifyPeer(false);
        $curlClient->setIgnoreErrors(true);
        $this->browser = new \Buzz\Browser($curlClient);
    }

    /**
     * The logic of the consumer
     *
     * @param \stdClass     $message
     * @return void
     */
    public function process($message) {
        $this->setMessage($message);
        $messageData = json_decode($message->body);

        try {
            $content = $this->getContent($this->browser, $messageData->url);
        } catch(\Exception $e) {
            $this->acknowledgeMessage($message);
            return;
        }

        $crawler = new Crawler($content);
        /* @var $crawler \Symfony\Component\DomCrawler\Crawler */

        $projectLinks = $crawler->filterXPath('//table[@class="project_list"]/tr/td[1]/a[@class="list"]');
        foreach ($projectLinks as $node) {
            $name = $node->nodeValue;

            $href = $node->getAttribute('href');
            $detailUrl = rtrim($messageData->url, '/') . $href;
            $content = $this->getContent($this->browser, $detailUrl);

            $detailCrawler = new Crawler($content);
            /* @var $detailCrawler \Symfony\Component\DomCrawler\Crawler */
            $gitUrl = $detailCrawler->filterXPath('//table[@class="projects_list"]/tr[@class="metadata_url"]/td[2]')->text();

            $gitwebRecord = $this->getGitwebFromDatabase($gitUrl);
            if ($gitwebRecord === false) {
                $id = $this->insertGitwebRecord($name, $gitUrl);
            } else {
                $id = $gitwebRecord['id'];
                $this->getLogger()->info('Gitweb record already exists', array('git' => $gitUrl));
            }<commit_msg>Catch exception in gitweb crawler during content requesting<commit_after><?php
/**
 * @todo adds a description (license text, description of this class / file, etc)
 */
namespace TYPO3Analysis\Consumer\Crawler;

use TYPO3Analysis\Consumer\ConsumerAbstract;
use Symfony\Component\DomCrawler\Crawler;

class Gitweb extends ConsumerAbstract {

    /**
     * HTTP Client
     *
     * @var \Buzz\Browser
     */
    protected $browser = null;

    /**
     * Gets a description of the consumer
     *
     * @return string
     */
    public function getDescription() {
        return 'Crawls a Gitweb-Index page for Git-repositories';
    }

    /**
     * Initialize the consumer.
     * Sets the queue and routing key
     *
     * @return void
     */
    public function initialize() {
        $this->setQueue('crawler.gitweb');
        $this->setRouting('crawler.gitweb');

        $curlClient = new \Buzz\Client\Curl();
        $curlClient->setVerifyPeer(false);
        $curlClient->setIgnoreErrors(true);
        $this->browser = new \Buzz\Browser($curlClient);
    }

    /**
     * The logic of the consumer
     *
     * @param \stdClass     $message
     * @return void
     */
    public function process($message) {
        $this->setMessage($message);
        $messageData = json_decode($message->body);

        try {
            $content = $this->getContent($this->browser, $messageData->url);
        } catch(\Exception $e) {
            $this->acknowledgeMessage($message);
            return;
        }

        $crawler = new Crawler($content);
        /* @var $crawler \Symfony\Component\DomCrawler\Crawler */

        $projectLinks = $crawler->filterXPath('//table[@class="project_list"]/tr/td[1]/a[@class="list"]');
        foreach ($projectLinks as $node) {
            $name = $node->nodeValue;

            $href = $node->getAttribute('href');
            $detailUrl = rtrim($messageData->url, '/') . $href;
            try {
                $content = $this->getContent($this->browser, $detailUrl);
            } catch (\Exception $e) {
                continue;
            }

            $detailCrawler = new Crawler($content);
            /* @var $detailCrawler \Symfony\Component\DomCrawler\Crawler */
            $gitUrl = $detailCrawler->filterXPath('//table[@class="projects_list"]/tr[@class="metadata_url"]/td[2]')->text();

            $gitwebRecord = $this->getGitwebFromDatabase($gitUrl);
            if ($gitwebRecord === false) {
                $id = $this->insertGitwebRecord($name, $gitUrl);
            } else {
                $id = $gitwebRecord['id'];
                $this->getLogger()->info('Gitweb record already exists', array('git' => $gitUrl));
            }
{'fraction_non_alphanumeric': 0.14770572620401973, 'fraction_numerical': 0.0015168752370117557, 'mean_word_length': 2.236196319018405, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2978203', 'n_tokens_mistral': 1573, 'n_tokens_neox': 1526, 'n_words': 298}