<commit_before><?php
class urlHelper extends mify {
	# Helper-class that extends upon mify
	# Here you'll find various functions that handles
	# url-verification and similar stuff
	# Splitted those up to avoid a bigass class with all
	# the functions inside it.
	
	public function __construct() {
		return true;
	}
	
	protected function verifyURL($url) {
		if(preg_match("^(http|https)\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,4}(\/\S*)?$^", $url) && $this->curlCheck($url)) {
			return true;
		}
		else {
			return false;
		}
	}
	
	protected static function curlCheck($url) {
		$matches = null;
		$p = parse_url($url);

		if($p == 0) {
			return false;
		}

		$c = curl_init();
		curl_setopt($c, CURLOPT_URL, $url);
		curl_setopt($c, CURLOPT_USERAGENT, 'Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0)');
		curl_setopt($c, CURLOPT_RETURNTRANSFER,1);
		curl_setopt($c, CURLOPT_FOLLOWLOCATION, 1);
		curl_setopt($c, CURLOPT_CONNECTTIMEOUT, 15);
		curl_setopt($c, CURLOPT_TIMEOUT, 20);
		curl_setopt($c, CURLOPT_NOBODY, true);
		curl_setopt($c, CURLOPT_HEADER, true);

		if($p['scheme'] == "https"){
			curl_setopt($c, CURLOPT_SSL_VERIFYHOST, 1);
			curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
		}
		 
		$res = curl_exec($c);
		curl_close($c);

		if(preg_match('/HTTP\/1\.\d+\s+(\d+)/', $res, $matches)) {
			$code = intval($matches[1]);
		}
		else {
			return false;
		}
		return (($code >= 200) && ($code < 400));
			
	}
}<commit_msg>Changed user-agent!<commit_after><?php
class urlHelper extends mify {
	# Helper-class that extends upon mify
	# Here you'll find various functions that handles
	# url-verification and similar stuff
	# Splitted those up to avoid a bigass class with all
	# the functions inside it.
	
	public function __construct() {
		return true; # Aw yeah :(
	}
	
	protected function verifyURL($url) {
		if(preg_match("^(http|https)\://[a-zA-Z0-9\-\.]+\.[a-zA-Z]{2,4}(\/\S*)?$^", $url) && $this->curlCheck($url)) {
			return true;
		}
		else {
			return false;
		}
	}
	
	protected static function curlCheck($url) {
		$matches = null;
		$p = parse_url($url);

		if($p == 0) {
			return false;
		}

		$c = curl_init();
		curl_setopt($c, CURLOPT_URL, $url);
		curl_setopt($c, CURLOPT_USERAGENT, 'Mozilla/4.0 (compatible; MSIE 5.01; Windows NT 5.0) mify-me/framework');
		curl_setopt($c, CURLOPT_RETURNTRANSFER,1);
		curl_setopt($c, CURLOPT_FOLLOWLOCATION, 1);
		curl_setopt($c, CURLOPT_CONNECTTIMEOUT, 15);
		curl_setopt($c, CURLOPT_TIMEOUT, 20);
		curl_setopt($c, CURLOPT_NOBODY, true);
		curl_setopt($c, CURLOPT_HEADER, true);

		if($p['scheme'] == "https"){
			curl_setopt($c, CURLOPT_SSL_VERIFYHOST, 1);
			curl_setopt($c, CURLOPT_SSL_VERIFYPEER, false);
		}
		 
		$res = curl_exec($c);
		curl_close($c);

		if(preg_match('/HTTP\/1\.\d+\s+(\d+)/', $res, $matches)) {
			$code = intval($matches[1]);
		}
		else {
			return false;
		}
		return (($code >= 200) && ($code < 400));
			
	}
}
{'fraction_non_alphanumeric': 0.17878057182225285, 'fraction_numerical': 0.018601446779193936, 'mean_word_length': 3.922033898305085, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '303036', 'n_tokens_mistral': 1346, 'n_tokens_neox': 1125, 'n_words': 240}