<commit_before><?php

	Class General{
		
		const CRLF = "\r\n";
		
		/***
		
		Method: sanitize
		Description: Will convert any special characters into their entity equivalents
		Param: $str - a string to operate on
		Return: the encoded version of the string
		
		***/
		public static function sanitize($str){
			return @htmlspecialchars($str);
		}
		
		/***
		
		Method: reverse_sanitize
		Description: Will revert any html entities to their character equivalents
		Param: $str - a string to operate on
		Return: the decoded version of the string
		
		***/		
		public static function reverse_sanitize($str){		 
		   return @htmlspecialchars_decode($str);
		}

		
		/***
		
		Method: validateString
		Description: will validate a string against a set of reqular expressions
		Param: $string - string to operate on
		       $rule - a single rule or array of rules
		Return: true or false
		
		***/
		public static function validateString($string, $rule){
		
			if(!is_array($rule) && $rule == '') return true;
			if(!is_array($string) && $string == '') return true;
			
			if(!is_array($rule)) $rule = array($rule);
			if(!is_array($string)) $string = array($string);
						
			foreach($rule as $r){
				foreach($string as $s){
					if(!preg_match($r, $s)) return false;
				}
			}
			return true;
		}
		
		public static function tabsToSpaces($string, $spaces=4){
			return str_replace("\t", str_pad(NULL, $spaces), $string);
		}

		/***
		
		Method: validateXML
		Description: This checks an xml document for well-formedness
		Param: $data - filename or xml document as a string
		       $errors - pointer to an array which will contain any validation errors
		       $isFile (optional) - if this is true, the method will attempt to read
		                            from a file ($data) instead.
			   $xsltProcessor (optional) - If set, the validation will be done using this
										   xslt processor rather than the built in XML parser
			   $encoding (optional) - If no XML header is expected, than this should be set to
			 						  match the encoding of the XML
		Return: true or false
		
		***/		
		public static function validateXML($data, &$errors, $isFile=true, $xsltProcessor=NULL, $encoding='UTF-8') {
			$_parser 	= null;
			$_data	 	= null;
			$_vals		= array();
			$_index		= array();
			
			if($isFile)
				$_data = @file_get_contents($data);
				
			else
				$_data = $data;

			$_data = preg_replace('/(&[\\w]{2,6};)/', '',$_data);
			$_data = preg_replace('/<!DOCTYPE[-.:"\'\/\\w\\s]+>/' , '', $_data);
			
			if(strpos($_data, '<?xml') === false){
				$_data = '<?xml version="1.0" encoding="'.$encoding.'"?><rootelement>'.$_data.'</rootelement>';
			}
			
			if(@is_object($xsltProcessor)){
				
				$xsl = '<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">

				<xsl:template match="/"></xsl:template>

				</xsl:stylesheet>';

				$xsltProcessor->process($_data, $xsl, array());

				if($xsltProcessor->isErrors()) {
					$errors = $xsltProcessor->getError(true);
					return false;
				}
				
			}else{
			
				$_parser = xml_parser_create();
				xml_parser_set_option($_parser, XML_OPTION_SKIP_WHITE, 0);
				xml_parser_set_option($_parser, XML_OPTION_CASE_FOLDING, 0);
				
				if(!@xml_parse($_parser, $_data)) {
					$errors = array('error' => xml_get_error_code($_parser) . ': ' . xml_error_string(xml_get_error_code($_parser)), 
									'col' => xml_get_current_column_number($_parser), 
									'line' => (xml_get_current_line_number($_parser) - 2));
					return false;
				}

				xml_parser_free($_parser);
			}
			
			return true;
			
		}
		

		/***
		
		Method: validateURL
		Description: will check that a string is a valid URL
		Param: $string - string to operate on
		Return: true or false
		
		***/		
		public static function validateURL($url){
			if($url != ''){
				if(!preg_match('#^http[s]?:\/\/#i', $url)){
					$url = 'http://' . $url;
				}
		
				if(!preg_match('/^[^\s:\/?#]+:(?:\/{2,3})?[^\s.\/?#]+(?:\.[^\s.\/?#]+)*(?:\/[^\s?#]*\??[^\s?#]*(#[^\s#]*)?)?$/', $url)){
					$url = '';
				}
			}
			
			return $url;
		}


		/***
		
		Method: cleanArray
		Description: Will strip any slashes from all array values
		Param: &$arr - pointer to an array to operate on. Can be multi-dimensional		
		<commit_msg>Fix Issue # 39. Only use URI regex once.<commit_after><?php

	Class General{
		
		const CRLF = "\r\n";
		
		/***
		
		Method: sanitize
		Description: Will convert any special characters into their entity equivalents
		Param: $str - a string to operate on
		Return: the encoded version of the string
		
		***/
		public static function sanitize($str){
			return @htmlspecialchars($str);
		}
		
		/***
		
		Method: reverse_sanitize
		Description: Will revert any html entities to their character equivalents
		Param: $str - a string to operate on
		Return: the decoded version of the string
		
		***/		
		public static function reverse_sanitize($str){		 
		   return @htmlspecialchars_decode($str);
		}

		
		/***
		
		Method: validateString
		Description: will validate a string against a set of reqular expressions
		Param: $string - string to operate on
		       $rule - a single rule or array of rules
		Return: true or false
		
		***/
		public static function validateString($string, $rule){
		
			if(!is_array($rule) && $rule == '') return true;
			if(!is_array($string) && $string == '') return true;
			
			if(!is_array($rule)) $rule = array($rule);
			if(!is_array($string)) $string = array($string);
						
			foreach($rule as $r){
				foreach($string as $s){
					if(!preg_match($r, $s)) return false;
				}
			}
			return true;
		}
		
		public static function tabsToSpaces($string, $spaces=4){
			return str_replace("\t", str_pad(NULL, $spaces), $string);
		}

		/***
		
		Method: validateXML
		Description: This checks an xml document for well-formedness
		Param: $data - filename or xml document as a string
		       $errors - pointer to an array which will contain any validation errors
		       $isFile (optional) - if this is true, the method will attempt to read
		                            from a file ($data) instead.
			   $xsltProcessor (optional) - If set, the validation will be done using this
										   xslt processor rather than the built in XML parser
			   $encoding (optional) - If no XML header is expected, than this should be set to
			 						  match the encoding of the XML
		Return: true or false
		
		***/		
		public static function validateXML($data, &$errors, $isFile=true, $xsltProcessor=NULL, $encoding='UTF-8') {
			$_parser 	= null;
			$_data	 	= null;
			$_vals		= array();
			$_index		= array();
			
			if($isFile)
				$_data = @file_get_contents($data);
				
			else
				$_data = $data;

			$_data = preg_replace('/(&[\\w]{2,6};)/', '',$_data);
			$_data = preg_replace('/<!DOCTYPE[-.:"\'\/\\w\\s]+>/' , '', $_data);
			
			if(strpos($_data, '<?xml') === false){
				$_data = '<?xml version="1.0" encoding="'.$encoding.'"?><rootelement>'.$_data.'</rootelement>';
			}
			
			if(@is_object($xsltProcessor)){
				
				$xsl = '<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">

				<xsl:template match="/"></xsl:template>

				</xsl:stylesheet>';

				$xsltProcessor->process($_data, $xsl, array());

				if($xsltProcessor->isErrors()) {
					$errors = $xsltProcessor->getError(true);
					return false;
				}
				
			}else{
			
				$_parser = xml_parser_create();
				xml_parser_set_option($_parser, XML_OPTION_SKIP_WHITE, 0);
				xml_parser_set_option($_parser, XML_OPTION_CASE_FOLDING, 0);
				
				if(!@xml_parse($_parser, $_data)) {
					$errors = array('error' => xml_get_error_code($_parser) . ': ' . xml_error_string(xml_get_error_code($_parser)), 
									'col' => xml_get_current_column_number($_parser), 
									'line' => (xml_get_current_line_number($_parser) - 2));
					return false;
				}

				xml_parser_free($_parser);
			}
			
			return true;
			
		}
		

		/***
		
		Method: validateURL
		Description: will check that a string is a valid URL
		Param: $string - string to operate on
		Return: a blank string or a valid URL
		
		***/		
		public static function validateURL($url){
			if($url != ''){
				if(!preg_match('#^http[s]?:\/\/#i', $url)){
					$url = 'http://' . $url;
				}
				
				include(TOOLKIT . '/util.validators.php');
				if(!preg_match($validators['URI'], $url)){
					$url = '';
				}
			}
			
			return $url;
		}


		/***
		
		Method: cleanArray
		Description: Will strip any slashes from all array values
		Param: &$arr - pointer to an array to operate on. Can be multi-dimensional		
		
{'fraction_non_alphanumeric': 0.1540880503144654, 'fraction_numerical': 0.0041928721174004195, 'mean_word_length': 2.8837630031659884, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 2, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 30}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '396649', 'n_tokens_mistral': 3462, 'n_tokens_neox': 2681, 'n_words': 751}