<commit_before><?php
/**
* ownCloud - News app
*
* @author <NAME>
* Copyright (c) 2012 - <NAME> <<EMAIL>>
*
* This file is licensed under the Affero General Public License version 3 or later.
* See the COPYING-README file
*
*/

// load SimplePie library
//TODO: is this file a suitable place for the following require?
 require_once('news/3rdparty/SimplePie/autoloader.php');

class OC_News_Utils {

	/**
	 * @brief Fetch a feed from remote
	 * @param url remote url of the feed
	 * @returns
	 */
	public static function fetch($url){
	//TODO: handle the case where fetching of the feed fails
		$spfeed = new SimplePie_Core();
		$spfeed->set_feed_url( $url );
		$spfeed->enable_cache( false );
		$spfeed->init();
		$spfeed->handle_content_type();
		$title = $spfeed->get_title();

		$spitems = $spfeed->get_items();
		$items = array();
		foreach($spitems as $spitem) { //FIXME: maybe we can avoid this loop
			$itemUrl = $spitem->get_permalink();
			$itemTitle = $spitem->get_title();
			$itemGUID = $spitem->get_id();
			$itemBody = $spitem->get_content();
			$items[] = new OC_News_Item($itemUrl, $itemTitle, $itemGUID, $itemBody);
		}

		$feed = new OC_News_Feed($url, $title, $items);

		$favicon = $spfeed->get_image_url();

		if ($favicon !== null) { // use favicon from feed
			if(OC_News_Utils::checkFavicon($favicon))
				$feed->setFavicon($favicon);
		}
		else { // try really hard to find a favicon
			if( null !== ($webFavicon = OC_News_Utils::discoverFavicon($url)) )
				$feed->setFavicon($webFavicon);
		}
		return $feed;
	}

	public static function checkFavicon($favicon) {
		$file = new SimplePie_File($favicon);

		//TODO additional checks? getimagesize()?
		if($file->success && strlen($file->body) > 0) {
			$sniffer = new SimplePie_Content_Type_Sniffer($file);
			if(substr($sniffer->get_type(), 0, 6) === 'image/') {
				return true;
			}
		}
		return false;
	}

	public static function discoverFavicon($url) {
		//try webroot favicon
		$favicon = SimplePie_Misc::absolutize_url('/favicon.ico', $url);

		if(OC_News_Utils::checkFavicon($favicon))
			return $favicon;

		//try to extract favicon from web page
		$absoluteUrl = SimplePie_Misc::absolutize_url('/', $url);

		$handle = curl_init ( );
		curl_setopt ( $handle, CURLOPT_URL, $absoluteUrl );
		curl_setopt ( $handle, CURLOPT_RETURNTRANSFER, 1 );
		curl_setopt ( $handle, CURLOPT_FOLLOWLOCATION, TRUE );
		curl_setopt ( $handle, CURLOPT_MAXREDIRS, 10 );

		if ( FALSE!==($page=curl_exec($handle)) ) {
			preg_match ( '/<[^>]*link[^>]*(rel=["\']icon["\']|rel=["\']shortcut icon["\']) .*href=["\']([^>]*)["\'].*>/iU', $page, $match );
			if (1<sizeof($match)) {
				// the specified uri might be an url, an absolute or a relative path
				// we have to turn it into an url to be able to display it out of context
				$favicon = htmlspecialchars_decode ( $match[2] );
				// test for an url<commit_msg>checkFavicon: also check for size<commit_after><?php
/**
* ownCloud - News app
*
* @author <NAME>
* Copyright (c) 2012 - <NAME> <<EMAIL>>
*
* This file is licensed under the Affero General Public License version 3 or later.
* See the COPYING-README file
*
*/

// load SimplePie library
//TODO: is this file a suitable place for the following require?
 require_once('news/3rdparty/SimplePie/autoloader.php');

class OC_News_Utils {

	/**
	 * @brief Fetch a feed from remote
	 * @param url remote url of the feed
	 * @returns
	 */
	public static function fetch($url){
	//TODO: handle the case where fetching of the feed fails
		$spfeed = new SimplePie_Core();
		$spfeed->set_feed_url( $url );
		$spfeed->enable_cache( false );
		$spfeed->init();
		$spfeed->handle_content_type();
		$title = $spfeed->get_title();

		$spitems = $spfeed->get_items();
		$items = array();
		foreach($spitems as $spitem) { //FIXME: maybe we can avoid this loop
			$itemUrl = $spitem->get_permalink();
			$itemTitle = $spitem->get_title();
			$itemGUID = $spitem->get_id();
			$itemBody = $spitem->get_content();
			$items[] = new OC_News_Item($itemUrl, $itemTitle, $itemGUID, $itemBody);
		}

		$feed = new OC_News_Feed($url, $title, $items);

		$favicon = $spfeed->get_image_url();

		if ($favicon !== null) { // use favicon from feed
			if(OC_News_Utils::checkFavicon($favicon))
				$feed->setFavicon($favicon);
		}
		else { // try really hard to find a favicon
			if( null !== ($webFavicon = OC_News_Utils::discoverFavicon($url)) )
				$feed->setFavicon($webFavicon);
		}
		return $feed;
	}

	public static function checkFavicon($favicon) {
		$file = new SimplePie_File($favicon);
		// size in bytes
		$filesize = strlen($file->body);

		if($file->success && $filesize > 0 && $filesize < 50000) {
			$sniffer = new SimplePie_Content_Type_Sniffer($file);
			if(substr($sniffer->get_type(), 0, 6) === 'image/') {
				return true;
			}
		}
		return false;
	}

	public static function discoverFavicon($url) {
		//try webroot favicon
		$favicon = SimplePie_Misc::absolutize_url('/favicon.ico', $url);

		if(OC_News_Utils::checkFavicon($favicon))
			return $favicon;

		//try to extract favicon from web page
		$absoluteUrl = SimplePie_Misc::absolutize_url('/', $url);

		$handle = curl_init ( );
		curl_setopt ( $handle, CURLOPT_URL, $absoluteUrl );
		curl_setopt ( $handle, CURLOPT_RETURNTRANSFER, 1 );
		curl_setopt ( $handle, CURLOPT_FOLLOWLOCATION, TRUE );
		curl_setopt ( $handle, CURLOPT_MAXREDIRS, 10 );

		if ( FALSE!==($page=curl_exec($handle)) ) {
			preg_match ( '/<[^>]*link[^>]*(rel=["\']icon["\']|rel=["\']shortcut icon["\']) .*href=["\']([^>]*)["\'].*>/iU', $page, $match );
			if (1<sizeof($match)) {
				// the specified uri might be an url, an absolute or a relative path
				// we have to turn it into an url to be able to display it out of context
				$favicon = htmlspecialchars_decode ( $match[2] );
				// test for an url
{'fraction_non_alphanumeric': 0.16612040710712436, 'fraction_numerical': 0.0056925996204933585, 'mean_word_length': 4.050522648083624, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5952725', 'n_tokens_mistral': 2329, 'n_tokens_neox': 2051, 'n_words': 520}