<commit_before><?

function is_url($url){
	$regex  = "((https?|ftp)\:\/\/)"; // SCHEME
	$regex .= "([a-z0-9+!*(),;?&=\$_.-]+(\:[a-z0-9+!*(),;?&=\$_.-]+)?@)?"; // User and Pass
	$regex .= "([a-z0-9-.]*)\.([a-z]{2,3}\/)"; // Host or IP
	$regex .= "(\:[0-9]{2,5})?"; // Port
	$regex .= "(([a-z0-9+\$_-]\.?)+)*\/?"; // Path
	$regex .= "(\?[a-z+&\$_.-][a-z0-9;:@&%=+\/\$_.-]*)?"; // GET Query
	$regex .= "(#[a-z_.-][a-z0-9+\$_.-]*)?"; // Anchor

	if(preg_match("/^$regex$/", $url)){
		return true;
	}
	else{
		return false;
	}

}

function validate_and_fix_url($url){


        $url_parts = parse_url($url);
	//print_r($url_parts);

	//BEGIN scheme and start of URL handling 
	$tmp_scheme_arr = explode('://',$url);
	if(count($tmp_scheme_arr) > 2){
		$new_url = '';
		foreach($tmp_scheme_arr AS $element){
			$element = trim($element);
			if($element != '' && $element != ':' && $element != '/' && $element != 'http' && $element != 'http:' && $element != 'https' && $element != 'https:'){
				$new_url .= $element;
				//echo '$element: '.$element."\n";
			}
		}
		$new_url   = $tmp_scheme_arr[0].'://'.$new_url;
		$new_url   = str_replace($tmp_scheme_arr[0].':///',$tmp_scheme_arr[0].'://',$new_url);
		$url_parts = parse_url($new_url);
		unset($new_url);
	}

	if(!isset($url_parts['scheme'])){
		if(strpos($url,'://') === 0){
			$url_parts         = parse_url('http'.$url);
			$new_url           = 'http://';
		}
		else{
			$url_parts         = parse_url('http://'.$url);
			$new_url           = 'http://';
		}
	}
	else{
		$tmp_scheme = strtolower($url_parts['scheme']);
		if($tmp_scheme != $url_parts['scheme']){
			$url_parts['scheme'] = $tmp_scheme;
		}
		$new_url = $tmp_scheme.'://';
	}

	if(preg_match("/^https?|ftp/iUs", $new_url) === false){
		$ret_arr['err'][]  = 'No scheme could be found!';
		$ret_arr['status'] = 'broken';
		return $ret_arr;
	}
	// END - scheme

	//user:pass
	if(isset($url_parts['user']) && isset($url_parts['pass'])){
		$new_url .= $url_parts['user'].':'.$url_parts['pass'].'@';
	}

	// beginn host handling
	if(!isset($url_parts['host'])){
		$ret_arr['err'][]  = 'No host could be identified!';
		$ret_arr['status'] = 'broken';
		return $ret_arr;
	}
	else{
		$tmp_host = strtolower($url_parts['host']);
		if($tmp_host != $url_parts['host']){
			$url_parts['host'] = $tmp_host;
		}
	}

	if(preg_match('/[^a-z0-9\.-]+/', $url_parts['host'],$matches)){
		//print_r($matches);
		//echo $url_parts['host']."\n";
		$ret_arr['err'][]  = 'Hostname ('.$url_parts['host'].') has illegal characters!';
		$ret_arr['status'] = 'broken';
		return $ret_arr;
	}

	$host_len = strlen($url_parts['host']);
	if($host_len > 255){
		$ret_arr['err'][]  = 'Hostname has more than 255 characters: '.$host_len.'!';
		$ret_arr['status'] = 'broken';
		return $ret_arr;<commit_msg>changed validate_and_fix_url() -> canonicalize_url() including new features and BUGFIX regexp in is_url()<commit_after><?php

function is_url($url){
	$regex  = "((https?|ftp)\:\/\/)"; // SCHEME
	$regex .= "([a-z0-9+!*(),;?&=\$_.-]+(\:[a-z0-9+!*(),;?&=\$_.-]+)?@)?"; // User and Pass
	//$regex .= "([a-z0-9-.]*)\.([a-z]{2,3}\/)"; // Host or IP
	$regex .= "([a-z0-9-\.]*)"; // Host or IP
	$regex .= "(\:[0-9]{2,5})?"; // Port
	//$regex .= "(([a-z0-9+\$_-]\.?)+)*\/?"; // Path
	$regex .= "\/([,%\.\/a-z0-9+\$_-])*?"; // Path
	$regex .= "(\?[a-z+&\$_.-][a-z0-9;:@&%=+\/\$_.-]*)?"; // GET Query
	$regex .= "(#[a-z_.-][a-z0-9+\$_.-]*)?"; // Anchor

	if(preg_match("/^$regex$/iUs", $url)){
		return true;
	}
	else{
		return false;
	}

}

function canonicalize_url($url,$target_scheme='http',$target_host=null){

	// handling of providet scheme and host
	if($target_scheme != strtolower($target_scheme)){
		trigger_error("WARNING in function canonicalize_url(): \$target_scheme should be lower case! \n", E_USER_WARNING);
		$target_scheme = strtolower($target_scheme);		
	}

	if($target_host != null && $target_host != strtolower($target_host)){
		trigger_error("WARNING in function canonicalize_url(): \$target_host should be lower case! \n", E_USER_WARNING);
		$target_host = strtolower($target_host);		
	}

	//initial url parsing
        $url_parts = parse_url($url);
	//print_r($url_parts);

	//BEGIN scheme and start of URL handling 
	$tmp_scheme_arr = explode('://',$url);
	if(count($tmp_scheme_arr) > 2){
		$new_url = '';
		foreach($tmp_scheme_arr AS $element){
			$element = trim($element);
			if($element != '' && $element != ':' && $element != '/' && $element != 'http' && $element != 'http:' && $element != 'https' && $element != 'https:'){
				$new_url .= $element;
				//echo '$element: '.$element."\n";
			}
		}
		$new_url   = $tmp_scheme_arr[0].'://'.$new_url;
		$new_url   = str_replace($tmp_scheme_arr[0].':///',$tmp_scheme_arr[0].'://',$new_url);
		$url_parts = parse_url($new_url);
		unset($new_url);
	}

	if(!isset($url_parts['scheme'])){
		if(strpos($url,'://') === 0){
			$url_parts         = parse_url($target_scheme.$url);
			$new_url           = $target_scheme.'://';
		}
		else{
			$url_parts         = parse_url($target_scheme.'://'.$url);
			$new_url           = $target_scheme.'://';
		}
	}
	else{
		$tmp_scheme = strtolower($url_parts['scheme']);
		if($tmp_scheme != $target_scheme){
			$url_parts['scheme'] = $target_scheme;
			$ret_arr['status'] = 'repaird';
		}
		else if($tmp_scheme != $url_parts['scheme']){
			$url_parts['scheme'] = $tmp_scheme;
			$ret_arr['status'] = 'repaird';
		}
		$new_url = $url_parts['scheme'].'://';
	}

	if(preg_match("/^https?|ftp\:\/\//iUs", $new_url) === false){
		$ret_arr['err'][]  = 'No scheme could be found!';
		$ret_arr['status'] = 'broken';
		return $ret_arr;
	}
	// END - scheme processing

	//user:pass
	if(isset($url_parts['user']) && isset($url_parts['pass'])){
		$new_url .= $url_parts['user'].':'.$url_parts['pass'].'@';
	}

	// beginn host handling
	if($target_host == null && !isset($url_parts['host'])){
		$ret_arr['err'][]  = 'No host could be identified!';
		$ret_arr['status'] = 'broken';
		return $ret_arr;
	}
	else{
		if(isset($url_parts['host'])){
			$tmp_host = strtolower($url_parts['host']);
		}
		else{
			$tmp_host = '';
		}

		if($target_host != null && $tmp_host != $target_host){
			$url_parts['host'] = $target_host;
			$ret_arr['status'] = 'repaird';
		}
		else if($tmp_host != $url_parts['host']){
			$url_parts['host'] = $tmp_host;
			$ret_arr['status'] = 'repaird';
		}
	}

	if(preg_match('/[^a-z0-9\.-]+/', $url_parts['host'],$matches)){
		//print_r($matches);
		//echo $url_parts['host']."\n";
		$ret_arr['err'][]  = 'Hostname ('.$url_parts['host'].') has illegal characters!';
		$ret_arr['status'] = 'broken';
		return $ret_arr;
	}

	$host_len = strlen($url_parts['host']);
	if($host_len > 255){
		$ret_arr['err'][]  = 'Hostname has more than 255 characters: '.$host_len.'!';
		$ret_arr['status'] = 'broken';
		return $ret_arr;
{'fraction_non_alphanumeric': 0.2756796473181484, 'fraction_numerical': 0.009698750918442321, 'mean_word_length': 3.732962447844228, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1166811', 'n_tokens_mistral': 2986, 'n_tokens_neox': 2774, 'n_words': 432}