<commit_before><?php

namespace Mini\Model\TypeCrawler;

use \Mini\Model\TypeCrawler\Storage\TypeCrawlerStorage;
use DOMDocument;
use DOMXpath;

class Pajbot extends TypeCrawler {
    /** @var int */
    protected static $crawlInterval = 604800;
    /** @var int */
    public static $type = 44;

    function __construct(TypeCrawlerStorage $storage) {
        parent::__construct($storage);
    }

    protected function doCrawl(): array {
        $url = "https://pajbot.com";

        $ch = curl_init($url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        $rawHTML = curl_exec($ch);
        curl_close($ch);

        $document = DOMDocument::loadHTML($rawHTML);

        $xpath = new DOMXpath($document);
        $elements = $xpath->query("*/div[@class='column pbot']");

        if(!empty($elementy)) {
            $ret = array();
            foreach($elements as $element) {
                $bot = new \stdClass;
                $bot->name = $element->getElementsByTabName('h2')->item(0)->textContent;
                $bot->type = $this::$type;
                $bot->channel = $xpath->query("*/div/a[starts-with(@href, 'http://twitch.tv/')]", $element)->textContent;
                $ret[] = $bot;
            }
            return $ret;
        }
        else {
            return array();
        }
    }
}<commit_msg>I don't like xpath<commit_after><?php

namespace Mini\Model\TypeCrawler;

use \Mini\Model\TypeCrawler\Storage\TypeCrawlerStorage;
use DOMDocument;
use DOMXpath;

class Pajbot extends TypeCrawler {
    /** @var int */
    protected static $crawlInterval = 604800;
    /** @var int */
    public static $type = 44;

    function __construct(TypeCrawlerStorage $storage) {
        parent::__construct($storage);
    }

    protected function doCrawl(): array {
        $url = "https://pajbot.com";

        $ch = curl_init($url);
        curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
        $rawHTML = curl_exec($ch);
        curl_close($ch);

        $document = DOMDocument::loadHTML($rawHTML);

        $xpath = new DOMXpath($document);
        $elements = $xpath->query("*/div[@class='column pbot']");

        if(!empty($elements)) {
            $ret = array();
            foreach($elements as $element) {
                $bot = new \stdClass;
                $bot->name = $element->getElementsByTabName('h2')->item(0)->textContent;
                $bot->type = $this::$type;
                $bot->channel = $xpath->query("div/a[starts-with(@href, 'http://twitch.tv/')]", $element)->textContent;
                $ret[] = $bot;
            }
            return $ret;
        }
        else {
            return array();
        }
    }
}
{'fraction_non_alphanumeric': 0.15703095859753824, 'fraction_numerical': 0.007459903021260724, 'mean_word_length': 2.1739644970414203, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '501184', 'n_tokens_mistral': 855, 'n_tokens_neox': 814, 'n_words': 153}