<commit_before><?php
namespace PVL\NewsAdapter;

class Rss extends AdapterAbstract
{
    public static function fetch($feed_url, $params = array())
    {
        $http_client = \Zend_Feed::getHttpClient();
        $http_client->setConfig(array('timeout' => 10));

        try
        {
            $news_feed = new \Zend_Feed_Rss($feed_url);
        }
        catch(\Exception $e)
        {
            return array();
        }

        if (!is_null($news_feed))
        {
            $latest_news = array();
            $article_num = 0;
            
            foreach ($news_feed as $item)
            {
                $article_num++;

                $guid = $item->guid();
                
                // Process categories.
                $categories_raw = (is_array($item->category)) ? $item->category : array($item->category);
                $categories = array();
                
                foreach($categories_raw as $category)
                {
                    $categories[] = $category->__toString();
                }
            
                // Process main description.
                $description = trim($item->description()); // Remove extraneous tags.
                $description = str_replace(array("\r", "\n"), array('', ' '), $description); // Strip new lines.
                $description = preg_replace('/[^(\x20-\x7F)]+/',' ', $description); // Strip "exotic" non-ASCII characters.
                $description = preg_replace('/<a[^(>)]+>read more<\/a>/i', '', $description); // Remove "read more" link.

                $web_url = $item->link();
                if (is_array($web_url))
                    $web_url = $web_url[0];

                if ($web_url instanceof \DOMElement)
                    $web_url = $web_url->nodeValue;

                if (!$web_url && substr($guid, 0, 4) == 'http')
                    $web_url = $guid;
                
                $news_item = array(
                    'guid'          => 'rss_'.md5($guid),
                    'timestamp'     => strtotime($item->pubDate()),
                    'title'         => $item->title(),
                    'body'          => $description,
                    'web_url'       => $web_url,
                    'author'        => $item->author(),
                );

                \PVL\Debug::print_r($item);
                \PVL\Debug::print_r($news_item);
                
                $latest_news[] = $news_item;
            }
<commit_msg>Handle multiple titles in RSS feeds (via <title> or <media-title>)<commit_after><?php
namespace PVL\NewsAdapter;

class Rss extends AdapterAbstract
{
    public static function fetch($feed_url, $params = array())
    {
        $http_client = \Zend_Feed::getHttpClient();
        $http_client->setConfig(array('timeout' => 10));

        try
        {
            $news_feed = new \Zend_Feed_Rss($feed_url);
        }
        catch(\Exception $e)
        {
            return array();
        }

        if (!is_null($news_feed))
        {
            $latest_news = array();
            $article_num = 0;
            
            foreach ($news_feed as $item)
            {
                $article_num++;

                $guid = $item->guid();

                $title = $item->title();

                if (is_array($title))
                    $title = $title[0]->nodeValue;
                
                // Process categories.
                $categories_raw = (is_array($item->category)) ? $item->category : array($item->category);
                $categories = array();
                
                foreach($categories_raw as $category)
                {
                    $categories[] = $category->__toString();
                }
            
                // Process main description.
                $description = trim($item->description()); // Remove extraneous tags.
                $description = str_replace(array("\r", "\n"), array('', ' '), $description); // Strip new lines.
                $description = preg_replace('/[^(\x20-\x7F)]+/',' ', $description); // Strip "exotic" non-ASCII characters.
                $description = preg_replace('/<a[^(>)]+>read more<\/a>/i', '', $description); // Remove "read more" link.

                $web_url = $item->link();
                if (is_array($web_url))
                    $web_url = $web_url[0];

                if ($web_url instanceof \DOMElement)
                    $web_url = $web_url->nodeValue;

                if (!$web_url && substr($guid, 0, 4) == 'http')
                    $web_url = $guid;
                
                $news_item = array(
                    'guid'          => 'rss_'.md5($guid),
                    'timestamp'     => strtotime($item->pubDate()),
                    'title'         => $title,
                    'body'          => $description,
                    'web_url'       => $web_url,
                    'author'        => $item->author(),
                );

                \PVL\Debug::print_r($item);
                \PVL\Debug::print_r($news_item);
                
                $latest_news[] = $news_item;
            }

{'fraction_non_alphanumeric': 0.14705310250923945, 'fraction_numerical': 0.004084808403034429, 'mean_word_length': 1.275221238938053, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 50, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5267820', 'n_tokens_mistral': 1431, 'n_tokens_neox': 1325, 'n_words': 255}