<commit_before><?php
	/**
	 * RSS consumer (because all "RSS" is not standard. Screw you, RSS.)
	 * @since Version 3.9.1
	 * @package Railpage
	 * @author <NAME>
	 */
	
	namespace Railpage\RSS;
	
	use SimpleXMLElement;
	use DOMDocument;
	use DOMXPath;
	use Exception;
	use DateTime;
	use DateTimeZone;
	use DateInterval;
	use GuzzleHttp\Client;
	use Railpage\AppCore;
	use Railpage\Url;
	use Railpage\Module;
	
	class Consume extends AppCore {
		
		/**
		 * RSS scraper user agent
		 * @since Version 3.9.1
		 * @const string SCRAPER_AGENT
		 */
		
		const SCRAPER_AGENT = "Railpage/Railpage";
		
		/**
		 * RSS scraper user agent version
		 * @since Version 3.9.1
		 * @const string SCRAPER_VERSION
		 */
		
		const SCRAPER_VERSION = "0.1";
		
		/**
		 * Feed name
		 * @since Version 3.9.1
		 * @var string $name
		 */
		
		public $name;
		
		/**
		 * Feed URL
		 * @since Version 3.9.1
		 * @var array $feeds
		 */
		
		private $feeds = array();
		
		/**
		 * Guzzle HTTP client
		 * @since Version 3.9.1
		 * @var object $GuzzleClient
		 */
		
		private $GuzzleClient;
		
		/**
		 * Constructor
		 * @since Version 3.9.1
		 * @param string $url
		 */
		
		public function __construct($url = false) {
			
			parent::__construct(); 
			
			$this->GuzzleClient = new Client;
			
			if ($url) {
				$this->addFeed($url);
			}
			
		}
		
		/**
		 * Add an RSS feed to this scraper
		 * @since Version 3.9.1
		 * @var string $url
		 * @return \Railpage\RSS\Consume
		 */
		
		public function addFeed($url = false) {
			if (!in_array($url, array_keys($this->feeds))) {
				$this->feeds[$url] = array(
					"url" => $url,
					"name" => "",
					"scraped" => "",
					"body" => "",
					"items" => array()
				);
			}
			
			return $this;
		}
		
		/**
		 * Scrape the RSS feed
		 * @since Version 3.9.1
		 * @return \Railpage\RSS\Consume
		 */
		
		public function scrape() {
			foreach ($this->feeds as $key => $feed) {
				$response = $this->GuzzleClient->get(
					$feed['url'], 
					array(
						"headers" => array(
							"User-Agent" => self::SCRAPER_AGENT . ' v.' . self::SCRAPER_VERSION
						),
						"timeout" => 60,
						"connect_timeout" => 5
					)
				);
				
				if ($response->getStatusCode() != 200) {
					throw new Exception(sprintf("Failed to scrape RSS feed %s: Error %s", $feed['url'], $response->getStatusCode()));
				}
				
				$this->feeds[$key]['body'] = $response->getBody(); 
			}
			
			return $this;
		}
		
		/**
		 * Parse through the scraped content and structure it
		 * @since Version 3.9.1
		 * @return \Railpage\RSS\Consume
		 */
		
		public function parse() {
			if (empty($this->feeds)) {
				throw new Exception("No scraped RSS data was found (hint: Consume::scrape())");
			}
			
			$items = array(); 
			
			foreach ($this->feeds as $feedsrc => $feed) {
				
				/**
				 * Create a new SimpleXMLElement object using the data returned from the RSS/Atom feed
				 */
				
				$xml = new SimpleXMLElement($feed['body'], LIBXML_NOCDATA);
				
				/**
				 * Find all namespaces
				 */
				
				$ns = $xml->getNamespaces(true);
				$type = "rss";
				
				if ($xml->channel->item != NULL) {
					$loop = $xml->channel->item; 
				} else {
					$type = "atom";
					$loop = $xml->entry;
				}
				
				/**
				 * Loop through each entry in the feed
				 */
				
				foreach ($loop as $node) {
					$date = !empty($node->date->__toString()) ? $node->date->__toString() : $node->pubDate->__toString();
					
					$link = $node->link->__toString();
					
					foreach ($node->link as $link) {
						if ($link['type'] == "text/html" || $link['rel'] == "alternate") {
							$link = $link['href']->__toString();
						}
					}
					
					$item = array(
						"id" => !empty($node->guid->__toString()) ? $node->guid->__toString() : $link,
						"title" => $node->title->__toString(),
						"description" => !empty($node->content->__toString()) ? $node->content->__toString() : $node->description->__toString(),
						"link" => $link,
						"tags" => $link,
						"category" => $link,
						"date" => $date,
					);
					
					/**
					 * Loop through all known namespaces and assemble the data
					 */
					
					foreach ($ns as $namespace) {
						foreach ($node->children($namespace) as $key => $data) {
							
							if (isset($item[$key])) {
								$item[$key] = trim($data->__toString()); 
							} elseif ($key == "encoded") {
								$item['description'] = $data->__toString();
							} else {
								$item['extra'][$key] = $data->__toString();
							}
						}
					}
					
					/**
					 * Organise the tag(s) situation a little better
					 */
					
					if (count($node->tag) > 0) {
						$tags = array(); 
						
						foreach ($node->tag as $tag) {
							$tags[] = $tag->__toString();
						}
						
						if (!empty(implode(",", $tags))) {
							$item['tags'] = implode(",", $tags);
						}
					}
					
					if (count($node->category) > 0) {
						$tags = array(); 
						
						foreach ($node->category as $tag) {
							$tags[] = $tag->__toString();<commit_msg>Minor change to RSS Consume<commit_after><?php
	/**
	 * RSS consumer (because all "RSS" is not standard. Screw you, RSS.)
	 * @since Version 3.9.1
	 * @package Railpage
	 * @author <NAME>
	 */
	
	namespace Railpage\RSS;
	
	use SimpleXMLElement;
	use DOMDocument;
	use DOMXPath;
	use Exception;
	use DateTime;
	use DateTimeZone;
	use DateInterval;
	use GuzzleHttp\Client;
	use Railpage\AppCore;
	use Railpage\Url;
	use Railpage\Module;
	
	class Consume extends AppCore {
		
		/**
		 * RSS scraper user agent
		 * @since Version 3.9.1
		 * @const string SCRAPER_AGENT
		 */
		
		const SCRAPER_AGENT = "Railpage/Railpage";
		
		/**
		 * RSS scraper user agent version
		 * @since Version 3.9.1
		 * @const string SCRAPER_VERSION
		 */
		
		const SCRAPER_VERSION = "0.1";
		
		/**
		 * Feed name
		 * @since Version 3.9.1
		 * @var string $name
		 */
		
		public $name;
		
		/**
		 * Feed URL
		 * @since Version 3.9.1
		 * @var array $feeds
		 */
		
		private $feeds = array();
		
		/**
		 * Guzzle HTTP client
		 * @since Version 3.9.1
		 * @var object $GuzzleClient
		 */
		
		private $GuzzleClient;
		
		/**
		 * Constructor
		 * @since Version 3.9.1
		 * @param string $url
		 */
		
		public function __construct($url = false) {
			
			parent::__construct(); 
			
			$this->GuzzleClient = new Client;
			
			if ($url) {
				$this->addFeed($url);
			}
			
		}
		
		/**
		 * Add an RSS feed to this scraper
		 * @since Version 3.9.1
		 * @var string $url
		 * @return \Railpage\RSS\Consume
		 */
		
		public function addFeed($url = false) {
			if (!in_array($url, array_keys($this->feeds))) {
				$this->feeds[$url] = array(
					"url" => $url,
					"name" => "",
					"scraped" => "",
					"body" => "",
					"items" => array()
				);
			}
			
			return $this;
		}
		
		/**
		 * Scrape the RSS feed
		 * @since Version 3.9.1
		 * @return \Railpage\RSS\Consume
		 */
		
		public function scrape() {
			foreach ($this->feeds as $key => $feed) {
				$response = $this->GuzzleClient->get(
					$feed['url'], 
					array(
						"headers" => array(
							"User-Agent" => self::SCRAPER_AGENT . ' v.' . self::SCRAPER_VERSION
						),
						"timeout" => 60,
						"connect_timeout" => 5
					)
				);
				
				if ($response->getStatusCode() != 200) {
					throw new Exception(sprintf("Failed to scrape RSS feed %s: Error %s", $feed['url'], $response->getStatusCode()));
				}
				
				$this->feeds[$key]['body'] = $response->getBody(); 
			}
			
			return $this;
		}
		
		/**
		 * Parse through the scraped content and structure it
		 * @since Version 3.9.1
		 * @return \Railpage\RSS\Consume
		 */
		
		public function parse() {
			if (empty($this->feeds)) {
				throw new Exception("No scraped RSS data was found (hint: Consume::scrape())");
			}
			
			$items = array(); 
			
			foreach ($this->feeds as $feedsrc => $feed) {
				
				/**
				 * Create a new SimpleXMLElement object using the data returned from the RSS/Atom feed
				 */
				
				$xml = new SimpleXMLElement($feed['body'], LIBXML_NOCDATA);
				
				/**
				 * Find all namespaces
				 */
				
				$ns = $xml->getNamespaces(true);
				$type = "rss";
				
				if ($xml->channel->item != NULL) {
					$loop = $xml->channel->item; 
				} else {
					$type = "atom";
					$loop = $xml->entry;
				}
				
				/**
				 * Loop through each entry in the feed
				 */
				
				foreach ($loop as $node) {
					$date = !empty($node->date->__toString()) ? $node->date->__toString() : $node->pubDate->__toString();
					
					$link = $node->link->__toString();
					
					foreach ($node->link as $link) {
						if ($link['type'] == "text/html" || $link['rel'] == "alternate") {
							$link = $link['href']->__toString();
						}
					}
					
					$item = array(
						"id" => !empty($node->guid->__toString()) ? $node->guid->__toString() : $link,
						"title" => $node->title->__toString(),
						"description" => !empty($node->content->__toString()) ? $node->content->__toString() : $node->description->__toString(),
						"link" => $link,
						"tags" => $link,
						"category" => $link,
						"date" => $date,
					);
					
					/**
					 * Loop through all known namespaces and assemble the data
					 */
					
					foreach ($ns as $namespace) {
						foreach ($node->children($namespace) as $key => $data) {
							
							if (isset($item[$key]) && !empty(trim($data->__toString()))) {
								$item[$key] = trim($data->__toString()); 
							} elseif ($key == "encoded") {
								$item['description'] = $data->__toString();
							} else {
								$item['extra'][$key] = $data->__toString();
							}
						}
					}
					
					/**
					 * Organise the tag(s) situation a little better
					 */
					
					if (count($node->tag) > 0) {
						$tags = array(); 
						
						foreach ($node->tag as $tag) {
							$tags[] = $tag->__toString();
						}
						
						if (!empty(implode(",", $tags))) {
							$item['tags'] = implode(",", $tags);
						}
					}
					
					if (count($node->category) > 0) {
						$tags = array(); 
						
						foreach ($node->category as $tag) {
							$tags[] = $tag->__toString();
{'fraction_non_alphanumeric': 0.17660482190693522, 'fraction_numerical': 0.00793729536660383, 'mean_word_length': 2.116883116883117, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 150, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5789048', 'n_tokens_mistral': 4738, 'n_tokens_neox': 3258, 'n_words': 782}