<commit_before><?php

// Favicon Parser for OneBox

if(isset($onebox)) {
	$properties = $onebox->availableProperties();

	$match = !in_array("favicon", $properties);

	if($match) {
		$data =  get_favicon($onebox);
		if($data) $onebox->update($data);
	}
}


function get_favicon($onebox) {
	// http://stackoverflow.com/questions/5701593/how-to-get-a-websites-favicon-with-php
	$data = array();
	$touchicon = parse_url($onebox->data['url'], PHP_URL_SCHEME)."://".parse_url($onebox->data['url'], PHP_URL_HOST)."/apple-touch-icon.png";
	$file_headers = @get_headers($touchicon);
	if($file_headers[0] == 'HTTP/1.1 404 Not Found') {
		$doc = $onebox->getDoc();
	    $xpath = new DOMXPath($doc);
		$xml = simplexml_import_dom($doc);
		if($xml) {
			$arr = $xml->xpath('//link[@rel="shortcut icon"]');
			if(isset($arr[0]['href'])) $data['favicon'] = (string)$arr[0]['href'];
			else {
				$arr = $xml->xpath('//link[@rel="icon"]');
				if(isset($arr[0]['href'])) $data['favicon'] = (string)$arr[0]['href'];
			}
		}
	}
	else {
	    $data['favicon'] = $touchicon;
	}

	return $data;
}<commit_msg>more robust method for checking for favicon<commit_after><?php

// Favicon Parser for OneBox

if(isset($onebox)) {
	$properties = $onebox->availableProperties();

	$match = !in_array("favicon", $properties);

	if($match) {
		$data =  get_favicon($onebox);
		if($data) $onebox->update($data);
	}
}


function get_favicon($onebox) {
	// http://stackoverflow.com/questions/5701593/how-to-get-a-websites-favicon-with-php
	$data = array();
	$touchicon = parse_url($onebox->data['url'], PHP_URL_SCHEME)."://".parse_url($onebox->data['url'], PHP_URL_HOST)."/apple-touch-icon.png";
	if(!checkRemoteFile($touchicon)) {
		$doc = $onebox->getDoc();
	    $xpath = new DOMXPath($doc);
		$xml = simplexml_import_dom($doc);
		if($xml) {
			$arr = $xml->xpath('//link[@rel="shortcut icon"]');
			if(isset($arr[0]['href'])) $data['favicon'] = (string)$arr[0]['href'];
			else {
				$arr = $xml->xpath('//link[@rel="icon"]');
				if(isset($arr[0]['href'])) $data['favicon'] = (string)$arr[0]['href'];
			}
		}
	}
	else {
	    $data['favicon'] = $touchicon;
	}

	return $data;
}

function checkRemoteFile($url)
{
    $ch = curl_init();
    curl_setopt($ch, CURLOPT_URL,$url);
    // don't download content
    curl_setopt($ch, CURLOPT_NOBODY, 1);
    curl_setopt($ch, CURLOPT_FAILONERROR, 1);
    curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1);
    if(curl_exec($ch)!==FALSE)
    {
        return true;
    }
    else
    {
        return false;
    }
}
{'fraction_non_alphanumeric': 0.22628026994839223, 'fraction_numerical': 0.012306470821754664, 'mean_word_length': 4.3276955602537, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 10}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6496624', 'n_tokens_mistral': 1066, 'n_tokens_neox': 967, 'n_words': 138}