<commit_before><?php
/* To-Do
https://developers.google.com/+/plugins/snippet/

<meta itemprop="name" content="Toller Titel">
<meta itemprop="description" content="Eine tolle Beschreibung">
<meta itemprop="image" content="http://maple.libertreeproject.org/images/tree-icon.png">

<body itemscope itemtype="http://schema.org/Product">
  <h1 itemprop="name">Shiny Trinket</h1>
  <img itemprop="image" src="{image-url}" />
  <p itemprop="description">Shiny trinkets are shiny.</p>
</body>
*/

if(!function_exists('deletenode')) {
	function deletenode(&$doc, $node)
	{
		$xpath = new DomXPath($doc);
		$list = $xpath->query("//".$node);
		foreach ($list as $child)
			$child->parentNode->removeChild($child);
	}
}

function completeurl($url, $scheme) {
        $urlarr = parse_url($url);

        if (isset($urlarr["scheme"]))
                return($url);

        $schemearr = parse_url($scheme);

        $complete = $schemearr["scheme"]."://".$schemearr["host"];

        if (@$schemearr["port"] != "")
                $complete .= ":".$schemearr["port"];

		if(strpos($urlarr['path'],'/') !== 0)
			$complete .= '/';

        $complete .= $urlarr["path"];

        if (@$urlarr["query"] != "")
                $complete .= "?".$urlarr["query"];

        if (@$urlarr["fragment"] != "")
                $complete .= "#".$urlarr["fragment"];

        return($complete);
}

function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) {

	$siteinfo = array();

	if ($count > 10) {
		logger("parseurl_getsiteinfo: Endless loop detected for ".$url, LOGGER_DEBUG);
		return($siteinfo);
	}

	$url = trim($url, "'");
	$url = trim($url, '"');
	$siteinfo["url"] = $url;
	$siteinfo["type"] = "link";

	$ch = curl_init();
	curl_setopt($ch, CURLOPT_URL, $url);
	curl_setopt($ch, CURLOPT_HEADER, 1);
	curl_setopt($ch, CURLOPT_NOBODY, 0);
	curl_setopt($ch, CURLOPT_TIMEOUT, 3);
	curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
	//curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
	//curl_setopt($ch,CURLOPT_USERAGENT,' Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0');
	curl_setopt($ch,CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; Friendica)");

	$header = curl_exec($ch);
	$curl_info = @curl_getinfo($ch);
        $http_code = $curl_info['http_code'];
	curl_close($ch);

	if ((($curl_info['http_code'] == "301") OR ($curl_info['http_code'] == "302") OR ($curl_info['http_code'] == "303") OR ($curl_info['http_code'] == "307"))
		AND (($curl_info['redirect_url'] != "") OR ($curl_info['location'] != ""))) {
		if ($curl_info['redirect_url'] != "")
			$siteinfo = parseurl_getsiteinfo($curl_info['redirect_url'], $no_guessing, $do_oembed, ++$count);
		else
			$siteinfo = parseurl_getsiteinfo($curl_info['location'], $no_guessing, $do_oembed, ++$count);
		return($siteinfo);
	}

	if ($do_oembed) {
		require_once("include/oembed.php");

		$oembed_data = oembed_fetch_url($url);

		if ($oembed_data->type != "error")
			$siteinfo["type"] = $oembed_data->type;
	}

	// Fetch the first mentioned charset. Can be in body or header
	$charset = "";
	if (preg_match('/charset=(.*?)['."'".'"\s\n]/', $header, $matches))
		$charset = trim(array_pop($matches));

	if ($charset == "")
		$charset = "utf-8";

	$pos = strpos($header, "\r\n\r\n");

	if ($pos)
		$body = trim(substr($header, $pos));
	else
		$body = $header;

	$body = mb_convert_encoding($body, "UTF-8", $charset);
	$body = mb_convert_encoding($body, 'HTML-ENTITIES', "UTF-8");

	$doc = new DOMDocument();
	@$doc->loadHTML($body);

	deletenode($doc, 'style');
	deletenode($doc, 'script');
	deletenode($doc, 'option');
	deletenode($doc, 'h1');
	deletenode($doc, 'h2');
	deletenode($doc, 'h3');
	deletenode($doc, 'h4');
	deletenode($doc, 'h5');
	deletenode($doc, 'h6');
	deletenode($doc, 'ol');
	deletenode($doc, 'ul');

	$xpath = new DomXPath($doc);

	$list = $xpath->query("//meta[@content]");
        foreach ($list as $node) {
                $attr = array();
                if ($node->attributes->length)
                        foreach ($node->attributes as $attribute)
                                $attr[$attribute->name] = $attribute->value;

                if (@$attr["http-equiv"] == 'refresh') {
                        $path = $attr["content"];
                        $pathinfo = explode(";", $path);
                        $content = "";<commit_msg>parse_url: Removing of warnings with invalid charsets<commit_after><?php
/* To-Do
https://developers.google.com/+/plugins/snippet/

<meta itemprop="name" content="Toller Titel">
<meta itemprop="description" content="Eine tolle Beschreibung">
<meta itemprop="image" content="http://maple.libertreeproject.org/images/tree-icon.png">

<body itemscope itemtype="http://schema.org/Product">
  <h1 itemprop="name">Shiny Trinket</h1>
  <img itemprop="image" src="{image-url}" />
  <p itemprop="description">Shiny trinkets are shiny.</p>
</body>
*/

if(!function_exists('deletenode')) {
	function deletenode(&$doc, $node)
	{
		$xpath = new DomXPath($doc);
		$list = $xpath->query("//".$node);
		foreach ($list as $child)
			$child->parentNode->removeChild($child);
	}
}

function completeurl($url, $scheme) {
        $urlarr = parse_url($url);

        if (isset($urlarr["scheme"]))
                return($url);

        $schemearr = parse_url($scheme);

        $complete = $schemearr["scheme"]."://".$schemearr["host"];

        if (@$schemearr["port"] != "")
                $complete .= ":".$schemearr["port"];

		if(strpos($urlarr['path'],'/') !== 0)
			$complete .= '/';

        $complete .= $urlarr["path"];

        if (@$urlarr["query"] != "")
                $complete .= "?".$urlarr["query"];

        if (@$urlarr["fragment"] != "")
                $complete .= "#".$urlarr["fragment"];

        return($complete);
}

function parseurl_getsiteinfo($url, $no_guessing = false, $do_oembed = true, $count = 1) {

	$siteinfo = array();

	if ($count > 10) {
		logger("parseurl_getsiteinfo: Endless loop detected for ".$url, LOGGER_DEBUG);
		return($siteinfo);
	}

	$url = trim($url, "'");
	$url = trim($url, '"');
	$siteinfo["url"] = $url;
	$siteinfo["type"] = "link";

	$ch = curl_init();
	curl_setopt($ch, CURLOPT_URL, $url);
	curl_setopt($ch, CURLOPT_HEADER, 1);
	curl_setopt($ch, CURLOPT_NOBODY, 0);
	curl_setopt($ch, CURLOPT_TIMEOUT, 3);
	curl_setopt($ch, CURLOPT_RETURNTRANSFER, true);
	//curl_setopt($ch, CURLOPT_FOLLOWLOCATION, 1);
	//curl_setopt($ch,CURLOPT_USERAGENT,' Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0');
	curl_setopt($ch,CURLOPT_USERAGENT, "Mozilla/5.0 (compatible; Friendica)");

	$header = curl_exec($ch);
	$curl_info = @curl_getinfo($ch);
        $http_code = $curl_info['http_code'];
	curl_close($ch);

	if ((($curl_info['http_code'] == "301") OR ($curl_info['http_code'] == "302") OR ($curl_info['http_code'] == "303") OR ($curl_info['http_code'] == "307"))
		AND (($curl_info['redirect_url'] != "") OR ($curl_info['location'] != ""))) {
		if ($curl_info['redirect_url'] != "")
			$siteinfo = parseurl_getsiteinfo($curl_info['redirect_url'], $no_guessing, $do_oembed, ++$count);
		else
			$siteinfo = parseurl_getsiteinfo($curl_info['location'], $no_guessing, $do_oembed, ++$count);
		return($siteinfo);
	}

	if ($do_oembed) {
		require_once("include/oembed.php");

		$oembed_data = oembed_fetch_url($url);

		if ($oembed_data->type != "error")
			$siteinfo["type"] = $oembed_data->type;
	}

	// Fetch the first mentioned charset. Can be in body or header
	$charset = "";
	if (preg_match('/charset=(.*?)['."'".'"\s\n]/', $header, $matches))
		$charset = trim(trim(trim(array_pop($matches)), ';,'));

	if ($charset == "")
		$charset = "utf-8";

	$pos = strpos($header, "\r\n\r\n");

	if ($pos)
		$body = trim(substr($header, $pos));
	else
		$body = $header;

	if (($charset != '') AND (strtoupper($charset) != "UTF-8")) {
		logger("parseurl_getsiteinfo: detected charset ".$charset, LOGGER_DEBUG);
		//$body = mb_convert_encoding($body, "UTF-8", $charset);
		$body = iconv($charset, "UTF-8//TRANSLIT", $body);
	}

	$body = mb_convert_encoding($body, 'HTML-ENTITIES', "UTF-8");

	$doc = new DOMDocument();
	@$doc->loadHTML($body);

	deletenode($doc, 'style');
	deletenode($doc, 'script');
	deletenode($doc, 'option');
	deletenode($doc, 'h1');
	deletenode($doc, 'h2');
	deletenode($doc, 'h3');
	deletenode($doc, 'h4');
	deletenode($doc, 'h5');
	deletenode($doc, 'h6');
	deletenode($doc, 'ol');
	deletenode($doc, 'ul');

	$xpath = new DomXPath($doc);

	$list = $xpath->query("//meta[@content]");
        foreach ($list as $node) {
                $attr = array();
                if ($node->attributes->length)
                        foreach ($node->attributes as $attribute)
                                $attr[$attribute->name] = $attribute->value;

                if (@$attr["http-equiv"] == 'refresh') {
                        $path = $attr["content"];
                        $pathinfo = explode(";", $path);
                        $content = "";
{'fraction_non_alphanumeric': 0.2166796570537802, 'fraction_numerical': 0.011802694577441266, 'mean_word_length': 3.7298578199052135, 'pattern_counts': {'":': 4, '<': 25, '<?xml version=': 0, '>': 49, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6566339', 'n_tokens_mistral': 3517, 'n_tokens_neox': 3227, 'n_words': 580}