<commit_before><?php

namespace Woe\CrawlerBundle;

class BilietaiCrawler
{
    const BILIETAI_ROOT_URL = "http://www.bilietai.lt/";

    private $page_index = 1;
    /* @var \DOMXPath $current_page */
    private $current_page;

    /**
     * Download current page and build DOM
     */
    public function fetchCurrentPage()
    {
        $url = self::BILIETAI_ROOT_URL . "lt/category/0/" . $this->page_index;
        $this->current_page = $this->fetchUrl($url);
    }

    /**
     * Download HTML of a page and build DOM
     *
     * @param $url
     * @return \DOMXPath
     */
    public function fetchUrl($url)
    {
        $html = file_get_contents($url);

        $dom = new \DOMDocument();
        @$dom->loadHTML($html);
        $domxpath = new \DOMXPath($dom);

        return $domxpath;
    }

    /**
     * Get list of event objects
     *
     * @return BilietaiEventParser[]
     */
    public function getEvents()
    {
        $events = [];
        $xpath = "//div[contains(@class, 'item_name')]/a[starts-with(@href, 'lt/event/')]";
        /* @var \DOMElement $node */
        foreach ($this->current_page->query($xpath) as $node) {
            $event_url = self::BILIETAI_ROOT_URL . $node->getAttribute("href");
            $event = new BilietaiEventParser($this->fetchUrl($event_url));

            if ($this->isValidEvent($event)) {
                $event->setSourceUrl($event_url);
                $events[] = $event;
            }
        }
        return $events;
    }

    /**
     * @param BilietaiEventParser $event
     * @return bool
     */
    private function isValidEvent($event)
    {
        return $event->isValid();
    }

    /**
     * Check whether the current page has 'Next' button
     *
     * @return bool
     */
    public function hasNextPage()
    {
        $xpath = "//*[contains(concat(' ', @class, ' '), ' next ')]";
        return $this->current_page && $this->current_page->query($xpath)->length !== 0;
    }

    /**
     * Increment current page index
     */
    public function nextPage()
    {
        $this->page_index += 1;
    }
}<commit_msg>Refactored BilietaiCrawler::getEvents()<commit_after><?php

namespace Woe\CrawlerBundle;

class BilietaiCrawler
{
    const BILIETAI_ROOT_URL = "http://www.bilietai.lt/";

    private $page_index = 1;
    /* @var \DOMXPath $current_page */
    private $current_page;

    /**
     * Download HTML of the current page and build DOM
     */
    public function fetchCurrentPage()
    {
        $url = $this->getCurrentPageUrl();
        $this->current_page = $this->fetchUrl($url);
    }

    /**
     * Download HTML of a page and build DOM
     *
     * @param $url
     * @return \DOMXPath
     */
    public function fetchUrl($url)
    {
        $html = file_get_contents($url);

        $dom = new \DOMDocument();
        @$dom->loadHTML($html);
        $domxpath = new \DOMXPath($dom);

        return $domxpath;
    }

    /**
     * Get list of event objects
     *
     * @return BilietaiEventParser[]
     */
    public function getEvents()
    {
        $events = [];
        $xpath = "//td[contains(concat(' ', @class, ' '), ' list_item ')]";

        /* @var \DOMElement $node */
        foreach ($this->getCurrentPage()->query($xpath) as $event_node) {
            if ($this->isOnSale($event_node)) {
                $event_url = $this->getEventUrl($event_node);
                $event_dom = $this->fetchUrl($event_url);
                $event = $this->getBilietaiEventParser($event_dom, $event_url);
                $events[] = $event;
            }
        }
        return $events;
    }

    /**
     * Check whether the current page has 'Next' button
     *
     * @return bool
     */
    public function hasNextPage()
    {
        $xpath = "//*[contains(concat(' ', @class, ' '), ' next ')]";
        return $this->getCurrentPage() && $this->getCurrentPage()->query($xpath)->length !== 0;
    }

    /**
     * Increment current page index
     */
    public function nextPage()
    {
        $this->page_index += 1;
    }

    /**
     * Get URL of the current page
     * @return string
     */
    public function getCurrentPageUrl()
    {
        return self::BILIETAI_ROOT_URL . "lt/category/0/" . $this->page_index;
    }

    /**
     * Get DOM of the current page
     * @return \DOMXPath
     */
    public function getCurrentPage()
    {
        return $this->current_page;
    }

    /**
     * Returns true if event tickets are still purchasable
     * @param \DOMNode $event_node
     * @return bool
     */
    public function isOnSale($event_node)
    {
        $xpath = "div[contains(@class, 'wf')]//div[contains(@class, 'price_padd')]/strong";
        $price_node = $this->getCurrentPage()->query($xpath, $event_node);
        $price = $price_node->length !== 0 ? $price_node->item(0)->nodeValue : null;
        return $price && strpos($price, 'parduota') === false;
    }

    /**
     * Extract URL of event from DOM node
     * @param \DOMNode $event_node
     * @return string
     */
    public function getEventUrl($event_node)
    {
        $xpath = "div[contains(@class, 'list_item_cont')]//a[starts-with(@href, 'lt/event/')]";
        $url_node = $this->getCurrentPage()->query($xpath, $event_node);
        $url = $url_node->length !== 0 ? $url_node->item(0)->getAttribute("href") : null;
        return self::BILIETAI_ROOT_URL . $url;
    }

    /**
     * Get BilietaiEventParser object
     * @param \DOMXpath $dom
     * @param $source_url
     * @return BilietaiEventParser
     */
    public function getBilietaiEventParser($dom, $source_url)
    {
        return new BilietaiEventParser($dom, $source_url);
    }
}
{'fraction_non_alphanumeric': 0.15287072912997535, 'fraction_numerical': 0.0021134202183867557, 'mean_word_length': 2.128925619834711, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '120373', 'n_tokens_mistral': 1837, 'n_tokens_neox': 1816, 'n_words': 365}