<commit_before><?php
class NumeramaBridge extends BridgeAbstract{

    public function loadMetadatas() {

        $this->maintainer = 'mitsukarenai';
        $this->name = 'Numerama';
        $this->uri = 'http://www.numerama.com/';
        $this->description = 'Returns the 5 newest posts from Numerama (full text)';
        $this->update = '2015-10-12';

    }

    public function collectData(array $param) {

        function NumeramaStripCDATA($string) {
            $string = str_replace('<![CDATA[', '', $string);
            $string = str_replace(']]>', '', $string);
            return $string;
        }

        function NumeramaExtractContent($url)
        {
            $html2 = file_get_html($url);
            $text = $html2->find('section[class=related-article]', 0)->innertext = ''; // remove related articles block
            $text = '<img alt="" style="max-width:300px;" src="'.$html2->find('meta[property=og:image]', 0)->getAttribute('content').'">'; // add post picture
            $text = $text.$html2->find('article[class=post-content]', 0)->innertext; // extract the post
            return $text;
        }

       $html = $this->file_get_html('http://www.numerama.com/feed/') or $this->returnError('Could not request Numerama.', 404);
        $limit = 0;

        foreach($html->find('item') as $element) {
            if($limit < 5) {
                $item = new \Item();
                $item->title = html_entity_decode(NumeramaStripCDATA($element->find('title', 0)->innertext));
                $item->author = NumeramaStripCDATA($element->find('dc:creator', 0)->innertext);
                $item->uri = NumeramaStripCDATA($element->find('guid', 0)->plaintext);
                $item->timestamp = strtotime($element->find('pubDate', 0)->plaintext);
                $item->content = NumeramaExtractContent($item->uri);
                $this->items[] = $item;
                $limit++;
            }
        }

    }

    public function getName() {
        return 'Numerama';
    }

    public function getURI() {
        return 'http://www.numerama.com/';
    }

    public function getCacheDuration() {
        return 1800; // 30min
    }
}<commit_msg>[Numerama] Use proxied file_get_html<commit_after><?php
class NumeramaBridge extends BridgeAbstract{

    public function loadMetadatas() {

        $this->maintainer = 'mitsukarenai';
        $this->name = 'Numerama';
        $this->uri = 'http://www.numerama.com/';
        $this->description = 'Returns the 5 newest posts from Numerama (full text)';
        $this->update = '2016-07-19';

    }

    public function collectData(array $param) {

        function NumeramaStripCDATA($string) {
            $string = str_replace('<![CDATA[', '', $string);
            $string = str_replace(']]>', '', $string);
            return $string;
        }

        function NumeramaExtractContent($bridge, $url)
        {
            $html2 = $bridge->file_get_html($url);
            $text = $html2->find('section[class=related-article]', 0)->innertext = ''; // remove related articles block
            $text = '<img alt="" style="max-width:300px;" src="'.$html2->find('meta[property=og:image]', 0)->getAttribute('content').'">'; // add post picture
            $text = $text.$html2->find('article[class=post-content]', 0)->innertext; // extract the post
            return $text;
        }

       $html = $this->file_get_html('http://www.numerama.com/feed/') or $this->returnError('Could not request Numerama.', 404);
        $limit = 0;

        foreach($html->find('item') as $element) {
            if($limit < 5) {
                $item = new \Item();
                $item->title = html_entity_decode(NumeramaStripCDATA($element->find('title', 0)->innertext));
                $item->author = NumeramaStripCDATA($element->find('dc:creator', 0)->innertext);
                $item->uri = NumeramaStripCDATA($element->find('guid', 0)->plaintext);
                $item->timestamp = strtotime($element->find('pubDate', 0)->plaintext);
                $item->content = NumeramaExtractContent($this, $item->uri);
                $this->items[] = $item;
                $limit++;
            }
        }

    }

    public function getName() {
        return 'Numerama';
    }

    public function getURI() {
        return 'http://www.numerama.com/';
    }

    public function getCacheDuration() {
        return 1800; // 30min
    }
}
{'fraction_non_alphanumeric': 0.17006340579710144, 'fraction_numerical': 0.015398550724637682, 'mean_word_length': 2.308614232209738, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 66, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1278248', 'n_tokens_mistral': 1371, 'n_tokens_neox': 1248, 'n_words': 243}