<commit_before><?php
class CNETBridge extends BridgeAbstract {

    private $topicName = '';

    public function loadMetadatas() {

        $this->maintainer = 'ORelio';
        $this->name = 'CNET News';
        $this->uri = 'http://www.cnet.com/';
        $this->description = 'Returns the newest articles. <br /> You may specify a topic found in some section URLs, else all topics are selected.';

        $this->parameters[] = array(
          'topic'=>array('name'=>'Topic name')
        );
    }

    public function collectData(){
        $param=$this->parameters[$this->queriedContext];

        function ExtractFromDelimiters($string, $start, $end) {
            if (strpos($string, $start) !== false) {
                $section_retrieved = substr($string, strpos($string, $start) + strlen($start));
                $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end));
                return $section_retrieved;
            } return false;
        }

        function StripWithDelimiters($string, $start, $end) {
            while (strpos($string, $start) !== false) {
                $section_to_remove = substr($string, strpos($string, $start));
                $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end));
                $string = str_replace($section_to_remove, '', $string);
            } return $string;
        }

        function CleanArticle($article_html) {
            $article_html = '<p>'.substr($article_html, strpos($article_html, '<p>') + 3);
            $article_html = StripWithDelimiters($article_html, '<span class="credit">', '</span>');
            $article_html = StripWithDelimiters($article_html, '<script', '</script>');
            $article_html = StripWithDelimiters($article_html, '<div class="shortcode related-links', '</div>');
            $article_html = StripWithDelimiters($article_html, '<a class="clickToEnlarge">', '</a>');
            return $article_html;
        }

        if (!empty($param['topic']['value']))
            $this->topicName = $param['topic']['value'];

        $pageUrl = 'http://www.cnet.com/'.(empty($this->topicName) ? '' : 'topics/'.$this->topicName.'/');
        $html = $this->getSimpleHTMLDOM($pageUrl) or $this->returnServerError('Could not request CNET: '.$pageUrl);
        $limit = 0;

        foreach($html->find('div.assetBody') as $element) {
            if ($limit < 8) {

                $article_title = trim($element->find('h2', 0)->plaintext);
                $article_uri = 'http://www.cnet.com'.($element->find('a', 0)->href);
                $article_timestamp = strtotime($element->find('time.assetTime', 0)->plaintext);
                $article_author = trim($element->find('a[rel=author]', 0)->plaintext);

                if (!empty($article_title) && !empty($article_uri) && strpos($article_uri, '/news/') !== false) {

                    $article_html = $this->getSimpleHTMLDOM($article_uri) or $this->returnServerError('Could not request CNET: '.$article_uri);

                    $article_content = trim(CleanArticle(ExtractFromDelimiters($article_html, '<div class="articleContent', '<footer>')));

                    $item = array();
                    $item['uri'] = $article_uri;
                    $item['title'] = $article_title;
                    $item['author'] = $article_author;
                    $item['timestamp'] = $article_timestamp;
                    $item['content'] = $article_content;
                    $this->items[] = $item;
                    $limit++;
                }
            }
        }
    }

    public function getName() {
        return 'CNET News Bridge'.(empty($this->topicName) ? '' : ' - '.$this->topicName);
    }

    public function getCacheDuration() {
        return 1800; // 30 minutes
    }
}
<commit_msg>[CNETBridge] remove useless member $topicName<commit_after><?php
class CNETBridge extends BridgeAbstract {

    public function loadMetadatas() {

        $this->maintainer = 'ORelio';
        $this->name = '<NAME>';
        $this->uri = 'http://www.cnet.com/';
        $this->description = 'Returns the newest articles. <br /> You may specify a topic found in some section URLs, else all topics are selected.';

        $this->parameters[] = array(
          'topic'=>array('name'=>'Topic name')
        );
    }

    public function collectData(){
        $param=$this->parameters[$this->queriedContext];

        function ExtractFromDelimiters($string, $start, $end) {
            if (strpos($string, $start) !== false) {
                $section_retrieved = substr($string, strpos($string, $start) + strlen($start));
                $section_retrieved = substr($section_retrieved, 0, strpos($section_retrieved, $end));
                return $section_retrieved;
            } return false;
        }

        function StripWithDelimiters($string, $start, $end) {
            while (strpos($string, $start) !== false) {
                $section_to_remove = substr($string, strpos($string, $start));
                $section_to_remove = substr($section_to_remove, 0, strpos($section_to_remove, $end) + strlen($end));
                $string = str_replace($section_to_remove, '', $string);
            } return $string;
        }

        function CleanArticle($article_html) {
            $article_html = '<p>'.substr($article_html, strpos($article_html, '<p>') + 3);
            $article_html = StripWithDelimiters($article_html, '<span class="credit">', '</span>');
            $article_html = StripWithDelimiters($article_html, '<script', '</script>');
            $article_html = StripWithDelimiters($article_html, '<div class="shortcode related-links', '</div>');
            $article_html = StripWithDelimiters($article_html, '<a class="clickToEnlarge">', '</a>');
            return $article_html;
        }

        $pageUrl = 'http://www.cnet.com/'.(empty($param['topic']['value']) ? '' : 'topics/'.$param['topic']['value'].'/');
        $html = $this->getSimpleHTMLDOM($pageUrl) or $this->returnServerError('Could not request CNET: '.$pageUrl);
        $limit = 0;

        foreach($html->find('div.assetBody') as $element) {
            if ($limit < 8) {

                $article_title = trim($element->find('h2', 0)->plaintext);
                $article_uri = 'http://www.cnet.com'.($element->find('a', 0)->href);
                $article_timestamp = strtotime($element->find('time.assetTime', 0)->plaintext);
                $article_author = trim($element->find('a[rel=author]', 0)->plaintext);

                if (!empty($article_title) && !empty($article_uri) && strpos($article_uri, '/news/') !== false) {

                    $article_html = $this->getSimpleHTMLDOM($article_uri) or $this->returnServerError('Could not request CNET: '.$article_uri);

                    $article_content = trim(CleanArticle(ExtractFromDelimiters($article_html, '<div class="articleContent', '<footer>')));

                    $item = array();
                    $item['uri'] = $article_uri;
                    $item['title'] = $article_title;
                    $item['author'] = $article_author;
                    $item['timestamp'] = $article_timestamp;
                    $item['content'] = $article_content;
                    $this->items[] = $item;
                    $limit++;
                }
            }
        }
    }

    public function getName() {
        $param=$this->parameters[$this->queriedContext];
        return 'CNET News Bridge'.(empty($param['topic']['value']) ? '' : ' - '.$this->param['topic']['value']);
    }

    public function getCacheDuration() {
        return 1800; // 30 minutes
    }
}

{'fraction_non_alphanumeric': 0.16835673426937078, 'fraction_numerical': 0.004160166406656267, 'mean_word_length': 2.440518783542039, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 78, 'https://': 0, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '475633', 'n_tokens_mistral': 2259, 'n_tokens_neox': 2121, 'n_words': 406}