<commit_before><?php 
// die Funktionalität von file_get_contents
// mit dem Curl-Modul umgesetzt
function file_get_contents_curl($url) {

	$ch = curl_init();
	
	curl_setopt($ch, CURLOPT_HEADER, 0);
	curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); //Set curl to return the data instead of printing it to the browser.
	curl_setopt($ch, CURLOPT_URL, $url);
	
	$data = curl_exec($ch);
	
	if(curl_getinfo($ch, CURLINFO_HTTP_CODE) != 200 and
	curl_getinfo($ch, CURLINFO_HTTP_CODE) != 304){
             $data = false;
	}
	
	curl_close($ch);
	
	return $data;
}

function is_url($url){
  if(substr_compare($url, 'http://', 0, 7) > 0 or substr_compare($url, 'https://', 0, 8) > 0 or substr_compare($url, 'ftp://', 0, 8) > 0){
      return true;
  }
  
  
  return false;
  
}


// Wrapper um file_get_contents
// Falls allow_url_fopen deaktiviert ist,
// wird CURL alls Fallback genutzt, falls vorhanden.
// Ansonsten wird false zurückgegeben.
function file_get_contents_wrapper($url){
   if(ini_get("allow_url_fopen")){
     return file_get_contents($url);
   } else {
     if(function_exists("curl_init") and is_url($url)){
       return file_get_contents_curl($url);
     }
     else
     {
        return false;    
     }
   }
}<commit_msg>file_get_contents_wrapper geht jetzt auch mit lokalen Pfaden<commit_after><?php 
// die Funktionalität von file_get_contents
// mit dem Curl-Modul umgesetzt
function file_get_contents_curl($url) {

	$ch = curl_init();
	
	curl_setopt($ch, CURLOPT_HEADER, 0);
	curl_setopt($ch, CURLOPT_RETURNTRANSFER, 1); //Set curl to return the data instead of printing it to the browser.
	curl_setopt($ch, CURLOPT_URL, $url);
	
	$data = curl_exec($ch);
	
	if(curl_getinfo($ch, CURLINFO_HTTP_CODE) != 200 and
	curl_getinfo($ch, CURLINFO_HTTP_CODE) != 304){
             $data = false;
	}
	
	curl_close($ch);
	
	return $data;
}

function is_url($url){
  if(substr_compare($url, 'http://', 0, 7) > 0 or substr_compare($url, 'https://', 0, 8) > 0 or substr_compare($url, 'ftp://', 0, 8) > 0){
      return true;
  }
  
  
  return false;
  
}


// Wrapper um file_get_contents
// Falls allow_url_fopen deaktiviert ist,
// wird CURL alls Fallback genutzt, falls vorhanden.
// Ansonsten wird false zurückgegeben.
function file_get_contents_wrapper($url){
   if(ini_get("allow_url_fopen") or !is_url($url)){
     return file_get_contents($url);
   } else {
     if(function_exists("curl_init") and is_url($url)){
       return file_get_contents_curl($url);
     } 

   }
   
   return false;
}
{'fraction_non_alphanumeric': 0.13572854291417166, 'fraction_numerical': 0.013572854291417165, 'mean_word_length': 3.746212121212121, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2116132', 'n_tokens_mistral': 991, 'n_tokens_neox': 901, 'n_words': 215}