<commit_before><?php

class Denkmal_Scraper_Source_Programmzeitung extends Denkmal_Scraper_Source_Abstract {

    public function run() {
        foreach ($this->_getDateList() as $date) {
            $dateStr = $date->format('d.m.Y');
            $url = 'http://www.programmzeitung.ch/index.cfm?Datum_von=' . $dateStr . '&Datum_bis=' .
                $dateStr . '&Rubrik=6&uuid=2BCD9733D9D9424C4EF093B3E35CB44B';
            $content = self::loadUrl($url);

            $this->processPageDate($content, $date);
        }
    }

    /**
     * @param string   $html
     * @param DateTime $date
     */
    public function processPageDate($html, DateTime $date) {
        $html = new CM_Dom_NodeList($html, true);
        $eventList = $html->find('.contentrahmen');
        /** @var CM_Dom_NodeList $event */
        foreach ($eventList as $event) {
            $eventText = $event->find('.veranstaltung');
            $locationName = $eventText->find('b')->getText();
            $description = $eventText->getChildren(XML_TEXT_NODE)->getText();

            $time = $event->find('.zeit')->getText();
            preg_match('#(\d+)\.(\d+)(\s+.\s+(\d+)\.(\d+))?#', $time, $matches);
            $from = new Denkmal_Scraper_Date($date);
            $from->setTime($matches[1], $matches[2]);

            $until = null;
            if (isset($matches[4]) && isset($matches[5])) {
                $until = clone $from;
                $until->setTime($matches[4], $matches[5]);
            }

            $this->_addEventAndVenue(
                $locationName,
                $description,
                $from->getDateTime(),
                $until ? $until->getDateTime() : null
            );
        }
    }
}
<commit_msg>Make programmzeitung scraper more robust<commit_after><?php

class Denkmal_Scraper_Source_Programmzeitung extends Denkmal_Scraper_Source_Abstract {

    public function run() {
        foreach ($this->_getDateList() as $date) {
            $dateStr = $date->format('d.m.Y');
            $url = 'http://www.programmzeitung.ch/index.cfm?Datum_von=' . $dateStr . '&Datum_bis=' .
                $dateStr . '&Rubrik=6&uuid=2BCD9733D9D9424C4EF093B3E35CB44B';
            $content = self::loadUrl($url);

            $this->processPageDate($content, $date);
        }
    }

    /**
     * @param string   $html
     * @param DateTime $date
     * @throws CM_Exception_Invalid
     */
    public function processPageDate($html, DateTime $date) {
        $html = new CM_Dom_NodeList($html, true);
        $eventList = $html->find('.contentrahmen');
        /** @var CM_Dom_NodeList $event */
        foreach ($eventList as $event) {
            $eventText = $event->find('.veranstaltung');
            $locationName = $eventText->find('b')->getText();
            $description = $eventText->getChildren(XML_TEXT_NODE)->getText();

            $time = $event->find('.zeit')->getText();
            if (0 === strlen(trim($time))) {
                continue; // Missing time
            }
            if (!preg_match('#(\d+)\.(\d+)(\s+.\s+(\d+)\.(\d+))?#', $time, $matches)) {
                throw new CM_Exception_Invalid('Cannot detect time from `' . $time . '`.');
            }
            $from = new Denkmal_Scraper_Date($date);
            $from->setTime($matches[1], $matches[2]);

            $until = null;
            if (isset($matches[4]) && isset($matches[5])) {
                $until = clone $from;
                $until->setTime($matches[4], $matches[5]);
            }

            $this->_addEventAndVenue(
                $locationName,
                $description,
                $from->getDateTime(),
                $until ? $until->getDateTime() : null
            );
        }
    }
}

{'fraction_non_alphanumeric': 0.16939158402573037, 'fraction_numerical': 0.014205306888233718, 'mean_word_length': 1.9903846153846154, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6541210', 'n_tokens_mistral': 1183, 'n_tokens_neox': 1137, 'n_words': 182}