<commit_before><?php

/**
 *  @Persist
 */
class Ultimahora extends Noticia
{
    public static function is($url)
    {
        return preg_match('/ultimahora\.com/', $url);
    }

    public static function search($text)
    {
        $page = Http::wget('http://www.ultimahora.com/contenidos/resultado.html?text=' . urlencode($text), 600);

        $urls = array();
        foreach ($page->query('//*[@class="result-obj"]//*[@class="cols1 clearfix"]//*[@class="t2"]//a') as $row) {
            $urls[] = ['url' => $row->getAttribute('href')];
        }
        foreach ($page->query('//*[@class="contenido"]//h3//a') as $row) {
            $urls[] = ['url' => $row->getAttribute('href')];
        }

        $url  = "http://www.ultimahora.com/_post/ultimahora/getMoreNewsBySearch.php";
        for ($i=1; ; $i++) {
            $args = [
                'text' => $text,
                'filterType' => '',
                'fechaDesde' => '',
                'fechaHasta' => '',
                'idCMSObjeto' => 47,
                'page' => 10,
                'id' => 212,
                'idCMSSeccion' => 195,
                'offset' => count($urls),
            ];

            $obj = Http::post($url, $args);
            foreach ($obj->query('//h3/a') as $path) {
                $urls[] = ['url' => $path->getAttribute('href')];
            }
            if ($obj->query('//h3/a')->length != 3) {
                break;
            }
        }

        return $urls;
    }

    public function crawl($v = 0)
    {
        if ($this->crawled && $this->version == 2) return;
        $xpath = Http::wget($this->url);

        $title  = Http::text($xpath->query('//h1'));
        if (empty($title)) {
            if ($v == 2) {
                /** givin' up */
                return;
            }
            $xpath = Http::wipe($this->url);
            return $this->crawl($v++);
        }
        $fecha  = Http::fecha($xpath->query('//*[@class="floatright"]'));
        $copete = Http::text($xpath->query('//*[@class="news-headline-obj"]'));
        $texto  = Http::text($xpath->query('//*[@class="newDetailTextChange"]'));

        $this->creado  = $fecha;
        $this->titulo  = $title;
        $this->texto   = $copete;
        $this->version = 2;
        $this->crawled = true;
        $this->crawled_data = compact('title', 'fecha', 'copete', 'texto');
    }
}<commit_msg>silly error<commit_after><?php

/**
 *  @Persist
 */
class Ultimahora extends Noticia
{
    public static function is($url)
    {
        return preg_match('/ultimahora\.com/', $url);
    }

    public static function search($text)
    {
        $page = Http::wget('http://www.ultimahora.com/contenidos/resultado.html?text=' . urlencode($text), 600);

        $urls = array();
        foreach ($page->query('//*[@class="result-obj"]//*[@class="cols1 clearfix"]//*[@class="t2"]//a') as $row) {
            $urls[] = ['url' => $row->getAttribute('href')];
        }
        foreach ($page->query('//*[@class="contenido"]//h3//a') as $row) {
            $urls[] = ['url' => $row->getAttribute('href')];
        }

        $url  = "http://www.ultimahora.com/_post/ultimahora/getMoreNewsBySearch.php";
        for ($i=1; ; $i++) {
            $args = [
                'text' => $text,
                'filterType' => '',
                'fechaDesde' => '',
                'fechaHasta' => '',
                'idCMSObjeto' => 47,
                'page' => 10,
                'id' => 212,
                'idCMSSeccion' => 195,
                'offset' => count($urls),
            ];

            $obj = Http::post($url, $args);
            foreach ($obj->query('//h3/a') as $path) {
                $urls[] = ['url' => $path->getAttribute('href')];
            }
            if ($obj->query('//h3/a')->length != 3) {
                break;
            }
        }

        return $urls;
    }

    public function crawl($v = 0)
    {
        if ($this->crawled && $this->version == 2) return;
        $xpath = Http::wget($this->url);

        $title  = Http::text($xpath->query('//h1'));
        if (empty($title)) {
            if ($v == 2) {
                /** givin' up */
                return;
            }
            $xpath = Http::wipe($this->url);
            return $this->crawl(++$v);
        }
        $fecha  = Http::fecha($xpath->query('//*[@class="floatright"]'));
        $copete = Http::text($xpath->query('//*[@class="news-headline-obj"]'));
        $texto  = Http::text($xpath->query('//*[@class="newDetailTextChange"]'));

        $this->creado  = $fecha;
        $this->titulo  = $title;
        $this->texto   = $copete;
        $this->version = 2;
        $this->crawled = true;
        $this->crawled_data = compact('title', 'fecha', 'copete', 'texto');
    }
}
{'fraction_non_alphanumeric': 0.21700879765395895, 'fraction_numerical': 0.010473397570171765, 'mean_word_length': 1.6825842696629214, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 73, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2727174', 'n_tokens_mistral': 1632, 'n_tokens_neox': 1562, 'n_words': 243}