<commit_before><?php
	/**
	 * @package toolkit
	 */
	/**
	 * General is a utility class that offers a number miscellaneous of
	 * functions that are used throughout Symphony.
	 */
	Class General{

		/**
		 * Convert any special characters into their entity equivalents. Since
		 * Symphony 2.3, this function assumes UTF-8 and will not double
		 * encode strings.
		 *
		 * @param string $source
		 *  a string to operate on.
		 * @return string
		 *  the encoded version of the string.
		 */
		public static function sanitize($source) {
			$source = htmlspecialchars($source, ENT_COMPAT, 'UTF-8', false);

			return $source;
		}

		/**
		 * Revert any html entities to their character equivalents.
		 *
		 * @param string $str
		 *  a string to operate on
		 * @return string
		 *  the decoded version of the string
		 */
		public static function reverse_sanitize($str){
		   return htmlspecialchars_decode($str, ENT_COMPAT);
		}

		/**
		 * Validate a string against a set of regular expressions.
		 *
		 * @param array|string $string
		 *  string to operate on
		 * @param array|string $rule
		 *  a single rule or array of rules
		 * @return boolean
		 *  false if any of the rules in $rule do not match any of the strings in
		 *  `$string`, return true otherwise.
		 */
		public static function validateString($string, $rule){

			if(!is_array($rule) && $rule == '') return true;
			if(!is_array($string) && $string == '') return true;

			if(!is_array($rule)) $rule = array($rule);
			if(!is_array($string)) $string = array($string);

			foreach($rule as $r){
				foreach($string as $s){
					if(!preg_match($r, $s)) return false;
				}
			}
			return true;
		}

		/**
		 * Replace the tabs with spaces in the input string.
		 *
		 * @param string $string
		 *  the string in which to replace the tabs with spaces.
		 * @param integer $spaces (optional)
		 *  the number of spaces to replace each tab with. This argument is optional
		 *  with a default of 4.
		 * @return string
		 *  the resulting string.
		 */
		public static function tabsToSpaces($string, $spaces=4){
			return str_replace("\t", str_pad(NULL, $spaces), $string);
		}

		/**
		 * Checks an xml document for well-formedness.
		 *
		 * @param string $data
		 *  filename, xml document as a string, or arbitrary string
		 * @param pointer &$errors
		 *  pointer to an array which will contain any validation errors
		 * @param boolean $isFile (optional)
		 *  if this is true, the method will attempt to read from a file, `$data`
		 *  instead.
		 * @param mixed $xsltProcessor (optional)
		 *  if set, the validation will be done using this XSLT processor rather
		 *  than the built in XML parser. the default is null.
		 * @param string $encoding (optional)
		 *  if no XML header is expected, than this should be set to match the
		 *  encoding of the XML
		 * @return boolean
		 *  true if there are no errors in validating the XML, false otherwise.
		 */
		public static function validateXML($data, &$errors, $isFile=true, $xsltProcessor=NULL, $encoding='UTF-8') {
			$_parser = null;
			$_data = null;
			$_vals = array();
			$_index = array();

			$_data = ($isFile) ? file_get_contents($data) : $data;

			$_data = preg_replace('/<!DOCTYPE[-.:"\'\/\\w\\s]+>/', NULL, $_data);

			if(strpos($_data, '<?xml') === false){
				$_data = '<?xml version="1.0" encoding="'.$encoding.'"?><rootelement>'.$_data.'</rootelement>';
			}

			if(is_object($xsltProcessor)){

				$xsl = '<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">

				<xsl:template match="/"></xsl:template>

				</xsl:stylesheet>';

				$xsltProcessor->process($_data, $xsl, array());

				if($xsltProcessor->isErrors()) {
					$errors = $xsltProcessor->getError(true);
					return false;
				}

			}
			else{

				$_parser = xml_parser_create();
				xml_parser_set_option($_parser, XML_OPTION_SKIP_WHITE, 0);
				xml_parser_set_option($_parser, XML_OPTION_CASE_FOLDING, 0);

				if(!xml_parse($_parser, $_data)){
					$errors = array('error' => xml_get_error_code($_parser) . ': ' . xml_error_string(xml_get_error_code($_parser)),
									'col' => xml_get_current_column_number($_parser),
									'line' => (xml_get_current_line_number($_parser) - 2));
					return false;
				}

				xml_parser_free($_parser);
			}

			return true;

		}

		/**
		 * Check that a string is a valid URL.
		 *
		 * @param string $url
		 *  string to operate on
		 * @return string
		 *  a blank string or a valid URL
		 */
		public static function validateURL($url = null){
			if(is_null($url) || $url == '') return $url;

			if(!preg_match('#^http[s]?:\/\/#i', $url)){
				$url = 'http://' . $url;
			}

			include(TOOLKIT . '/util.validators.php');
			if(!preg_match($validators['URI'], $url)){
				$url = '';
			}

			return $url;
		}

		/**
		 * Strip any slashes from all array values.
		 *
		 * @param array &$arr
		 *  Pointer to an array to operate on. Can be multi-dimensional.
		 */
		public static function cleanArray(array &$arr) {
			foreach($arr as $k => $v){
				if(is_array($v))
					self::cleanArray($arr[$k]);
				else
					$arr[$k] = stripslashes($v);
			}
		}

		/**
		 * Flatten the input array. Any elements of the input array that are
		 * themselves arrays will be removed and the contents of the removed array
		 * inserted in its place. The keys for the inserted values will be the
		 * concatenation of the keys in the original arrays in which it was embedded.
		 * The elements of the path are separated by periods (.). For example,
		 * given the following nested array structure:
		 * `
		 * array(1 =>
		 *  		array('key' => 'value'),
		 *  	2 =>
		 *  		array('key2' => 'value2', 'key3' => 'value3')
		 *  	)
		 * `
		 * will flatten to:
		 * `array('1.key' => 'value', '2.key2' => 'value2', '2.key3' => 'value3')`
		 *
		 * @param array &$source
		 *  The array to flatten, passed by reference
		 * @param array &$output (optional)
		 *  The array in which to store the flattened input, passed by reference.
		 *  if this is not provided then a new array will be created.
		 * @param string $path (optional)
		 *  the current prefix of the keys to insert into the output array. this
		 *  defaults to null.
		 */
		public static function flattenArray(array &$source, &$output = null, $path = null) {
			if (is_null($output)) $output = array();

			foreach ($source as $key => $value) {
				if (is_int($key)) $key = (string)($key + 1);
				if (!is_null($path)) $key = $path . '.' . (string)$key;

				if (is_array($value)) self::flattenArray($value, $output, $key);
				else $output[$key] = $value;
			}

			$source = $output;
		}

		/**
		 * Flatten the input array. Any elements of the input array that are
		 * themselves arrays will be removed and the contents of the removed array
		 * inserted in its place. The keys for the inserted values will be the
		 * concatenation of the keys in the original arrays in which it was embedded.
		 * The elements of the path are separated by colons (:). For example, given
		 * the following nested array structure:
		 * `
		 * array(1 =>
		 *  		array('key' => 'value'),
		 *  	2 =>
		 *  		array('key2' => 'value2', 'key3' => 'value3')
		 *  	)
		 * `
		 * will flatten to:
		 * `array('1:key' => 'value', '2:key2' => 'value2', '2:key3' => 'value3')`
		 *
		 *
		 * @param array &$output
		 *  The array in which to store the flattened input, passed by reference.
		 * @param array &$source
		 *  The array to flatten, passed by reference
		 * @param string $path
		 *  the current prefix of the keys to insert into the output array.
		 */
		protected static function flattenArraySub(array &$output, array &$source, $path) {
			foreach ($source as $key => $value) {
				$key = $path . ':' . $key;

				if (is_array($value)) self::flattenArraySub($output, $value, $key);
				else $output[$key] = $value;
			}
		}

		/**
		 * Allows you to send emails. It initializes the core email class.
		 *
		 * @deprecated Since Symphony 2.2
		 * @param string $to_email
		 *  email of the recipient
		 * @param string $from_email
		 *  the from email address. This is usually your email
		 * @param string $from_name
		 *  the name of the sender
		 * @param string $subject
		 *  subject of the email
		 * @param string $message
		 *  contents of the email
		 * @param array $additional_headers
		 *  an array containing additional email headers. This will NOT work
		 *  for Content-Type header fields which will be added/overwritten by
		 *  the email gateways.)
		 * @return boolean
		 *  true on success
		 */
		public static function sendEmail($to_email, $from_email, $from_name, $subject, $message, array $additional_headers = array()) {

			try{
				$email = Email::create();

				if (!empty($additional_headers)) {
					foreach ($additional_headers as $name => $body) {
						$email->appendHeaderField($name, $body);
					}
				}
				$email->sender_name = $from_name;
				$email->sender_email_address = $from_email;

				$email->recipients = $email->setRecipients($to_email);
				$email->text_plain = $message;
				$email->subject = $subject;

				return $email->send();
			}
			catch(EmailGatewayException $e){
				throw new SymphonyErrorPage('Error sending email. ' . $e->getMessage());
			}
			catch(EmailException $e){
				throw new SymphonyErrorPage('Error sending email. ' . $e->getMessage());
			}
		}

		/**
		 * Given a string, this will clean it for use as a Symphony handle. Preserves multi-byte characters.
		 *
		 * @since Symphony 2.2.1
		 * @param string $string
		 *  String to be cleaned up
		 * @param int $max_length
		 *  The maximum number of characters in the handle
		 * @param string $delim
		 *  All non-valid characters will be replaced with this
		 * @param boolean $uriencode
		 *  Force the resultant string to be uri encoded making it safe for URLs
		 * @param array $additional_rule_set
		 *  An array of REGEX patterns that should be applied to the `$string`. This
		 *  occurs after the string has been trimmed and joined with the `$delim`
		 * @return string
		 *  Returns resultant handle
		 */
		public static function createHandle($string, $max_length=255, $delim='-', $uriencode=false, $additional_rule_set=NULL) {

			$max_length = intval($max_length);

			// Strip out any tag
			$string = strip_tags($string);

			// Remove punctuation
			$string = preg_replace('/[\\.\'"]+/', NULL, $string);

			// Trim it
			if($max_length > 0) $string = General::limitWords($string, $max_length);

			// Replace spaces (tab, newline etc) with the delimiter
			$string = preg_replace('/[\s]+/', $delim, $string);

			// Find all legal characters
			preg_match_all('/[^<>?@:!-\/\[-`;‘’…]+/u', $string, $matches);

			// Join only legal character with the $delim
			$string = implode($delim, $matches[0]);

			// Allow for custom rules
			if(is_array($additional_rule_set) && !empty($additional_rule_set)) {
				foreach($additional_rule_set as $rule => $replacement) $string = preg_replace($rule, $replacement, $string);
			}

			// Remove leading or trailing delim characters
			$string = trim($string, $delim);

			// Encode it for URI use
			if($uriencode) $string = urlencode($string);

			// Make it lowercase
			$string = strtolower($string);

			return $string;

		}

		/**
		 * Given a string, this will clean it for use as a filename. Preserves multi-byte characters.
		 *
		 * @since Symphony 2.2.1
		 * @param string $string
		 *  String to be cleaned up
		 * @param string $delim
		 *  All non-valid characters will be replaced with this
		 * @return string
		 *  Returns created filename
		 */
		public static function createFilename($string, $delim='-') {

			// Strip out any tag
			$string = strip_tags($string);

			// Find all legal characters
			$count = preg_match_all('/[\p{L}\w:;.,+=~]+/u', $string, $matches);
			if($count <= 0 || $count == false) {
				preg_match_all('/[\w:;.,+=~]+/', $string, $matches);
			}

			// Join only legal character with the $delim
			$string = implode($delim, $matches[0]);

			// Remove leading or trailing delim characters
			$string = trim($string, $delim);

			// Make it lowercase
			$string = strtolower($string);

			return $string;

		}

		/**
		 * Extract the first `$val` characters of the input string. If `$val`
		 * is larger than the length of the input string then the original
		 * input string is returned.
		 *
		 * @param string $str
		 *  the string to operate on
		 * @param integer $val
		 *  the number to compare lengths with
		 * @return string|boolean
		 *  the resulting string or false on failure.
		 */
		public static function substrmin($str, $val){
			return(substr($str, 0, min(strlen($str), $val)));
		}

		/**
		 * Extract the first `$val` characters of the input string. If
		 * `$val` is larger than the length of the input string then
		 * the original input string is returned
		 *
		 * @param string $str
		 *  the string to operate on
		 * @param integer $val
		 *  the number to compare lengths with
		 * @return string|boolean
		 *  the resulting string or false on failure.
		 */
		public static function substrmax($str, $val){
			return(substr($str, 0, max(strlen($str), $val)));
		}

		/**
		 * Extract the last `$num` characters from a string.
		 *
		 * @param string $str
		 *  the string to extract the characters from.
		 * @param integer $num
		 *  the number of characters to extract.
		 * @return string|boolean
		 *  a string containing the last `$num` characters of the
		 *  input string, or false on failure.
		 */
		public static function right($str, $num){
			$str = substr($str, strlen($str)-$num,  $num);
			return $str;
		}

		/**
		 * Extract the first `$num` characters from a string.
		 *
		 * @param string $str
		 *  the string to extract the characters from.
		 * @param integer $num
		 *  the number of characters to extract.
		 * @return string|boolean
		 *  a string containing the last `$num` characters of the
		 *  input string, or false on failure.
		 */
		public static function left($str, $num){
			$str = substr($str, 0, $num);
			return $str;
		}

		/**
		 * Create all the directories as specified by the input path. If the current
		 * directory already exists, this function will return true.
		 *
		 * @param string $path
		 *  the path containing the directories to create.
		 * @param integer $mode (optional)
		 *  the permissions (in octal) of the directories to create. Defaults to 0755
		 * @return boolean
		 */
		public static function realiseDirectory($path, $mode=0755){
			if(is_dir($path)) return true;

			$current_umask = umask(0);
			$success = mkdir($path, intval($mode, 8), true);
			umask($current_umask);

			return $success;
		}

		/**
		 * Recursively deletes all files and directories given a directory. This
		 * function has two path. This function optionally takes a `$silent` parameter,
		 * which when `false` will throw an `Exception` if there is an error deleting a file
		 * or folder.
		 *
		 * @since Symphony 2.3
		 * @param string $dir
		 *  the path of the directory to delete
		 * @param boolean $silent (optional)
		 *  true if an exception should be raised if an error occurs, false
		 *  otherwise. this defaults to true.
		 * @return boolean
		 */
		public static function deleteDirectory($dir, $silent = true) {
			try {
				if (!file_exists($dir)) return true;

				if (!is_dir($dir)) return unlink($dir);

				foreach (scandir($dir) as $item) {
					if ($item == '.' || $item == '..') continue;

					if (!self::deleteDirectory($dir.DIRECTORY_SEPARATOR.$item)) return false;
				}

				return rmdir($dir);
			}
			catch(Exception $ex) {
				if($silent === false){
					throw new Exception(__('Unable to remove - %s', array($dir)));
				}

				return false;
			}
		}

		/**
		 * Search a multi-dimensional array for a value.
		 *
		 * @param mixed $needle
		 *  the value to search for.
		 * @param array $haystack
		 *  the multi-dimensional array to search.
		 * @return boolean
		 *  true if `$needle` is found in `$haystack`.
		 *  true if `$needle` == `$haystack`.
		 *  true if `$needle` is found in any of the arrays contained within `$haystack`.
		 *  false otherwise.
		 */
		public static function in_array_multi($needle, $haystack){

			if($needle == $haystack) return true;

			if(is_array($haystack)){
				foreach($haystack as $key => $val){

					if(is_array($val)){
						if(self::in_array_multi($needle, $val)) return true;
					}

					elseif(!strcmp($needle, $key) || !strcmp($needle, $val)){
						return true;
					}
				}
			}

			return false;
		}

		/**
		 * Search an array for multiple values.
		 *
		 * @param array $needles
		 *  the values to search the `$haystack` for.
		 * @param array $haystack
		 *  the in which to search for the `$needles`
		 * @return boolean
		 *  true if any of the `$needles` are in `$haystack`,
		 *  false otherwise.
		 */
		public static function in_array_all($needles, $haystack){

			foreach($needles as $n){
				if(!in_array($n, $haystack)) return false;
			}

			return true;
		}

		/**
		 * Transform a multi-dimensional array to a flat array. The input array
		 * is expected to conform to the structure of the `$_FILES` variable.
		 *
		 * @param array $filedata
		 *  the raw `$_FILES` data structured array
		 * @return array
		 *  the flattened array.
		 */
		public static function processFilePostData($filedata){

			$result = array();

			foreach($filedata as $key => $data){
				foreach($data as $handle => $value){
					if(is_array($value)){
						foreach($value as $index => $pair){

							if(!is_array($result[$handle][$index])) $result[$handle][$index] = array();

							if(!is_array($pair)) $result[$handle][$index][] = $pair;
							else $result[$handle][$index][array_pop(array_keys($pair))][$key] = array_pop(array_values($pair));
						}
					}

					else $result[$handle][$key] = $value;
				}
			}

			return $result;
		}

		/**
		 * Merge `$_POST` with `$_FILES` to produce a flat array of the contents
		 * of both. If there is no merge_file_post_data function defined then
		 * such a function is created. This is necessary to overcome PHP's ability
		 * to handle forms. This overcomes PHP's convoluted `$_FILES` structure
		 * to make it simpler to access `multi-part/formdata`.
		 *
		 * @return array
		 *  a flat array containing the flattened contents of both `$_POST` and
		 *  `$_FILES`.
		 */
		public static function getPostData() {
			if (!function_exists('merge_file_post_data')) {
				function merge_file_post_data($type, array $file, &$post) {
					foreach ($file as $key => $value) {
						if (!isset($post[$key])) $post[$key] = array();
						if (is_array($value)) merge_file_post_data($type, $value, $post[$key]);
						else $post[$key][$type] = $value;
					}
				}
			}

			$files = array(
				'name'		=> array(),
				'type'		=> array(),
				'tmp_name'	=> array(),
				'error'		=> array(),
				'size'		=> array()
			);
			$post = $_POST;

			if(is_array($_FILES) && !empty($_FILES)){
				foreach ($_FILES as $key_a => $data_a) {
					if(!is_array($data_a)) continue;
					foreach ($data_a as $key_b => $data_b) {
						$files[$key_b][$key_a] = $data_b;
					}
				}
			}

			foreach ($files as $type => $data) {
				merge_file_post_data($type, $data, $post);
			}

			return $post;
		}

		/**
		 * Find the next available index in an array. Works best with numeric keys.
		 * The next available index is the minimum integer such that the array does
		 * not have a mapping for that index. Uses the increment operator on the
		 * index type of the input array, whatever that may do.
		 *
		 * @param array $array
		 *  the array to find the next index for.
		 * @param mixed $seed (optional)
		 *  the object with which the search for an empty index is initialized. this
		 *  defaults to null.
		 * @return integer
		 *  the minimum empty index into the input array.
		 */
		public static function array_find_available_index($array, $seed=NULL){

			if(!is_null($seed)) $index = $seed;
			else{
				$keys = array_keys($array);
				sort($keys);
				$index = array_pop($keys);
			}

			if(isset($array[$index])){
				do{
					$index++;
				}while(isset($array[$index]));
			}

			return $index;
		}

		/**
		 * Filter the duplicate values from an array into a new array, optionally
		 * ignoring the case of the values (assuming they are strings?). A new array
		 * is returned, the input array is left unchanged.
		 *
		 * @param array $array
		 *  the array to filter.
		 * @param boolean $ignore_case
		 *  true if the case of the values in the array should be ignored, false otherwise.
		 * @return
		 *  a new array containing only the unique elements of the input array.
		 */
		public static function array_remove_duplicates(array $array, $ignore_case=false){
			return ($ignore_case == true ? self::array_iunique($array) : array_unique($array));
		}

		/**
		 * Test whether a value is in an array based on string comparison, ignoring
		 * the case of the values.
		 *
		 * @param mixed $needle
		 *  the object to search the array for.
		 * @param array $haystack
		 *  the array to search for the `$needle`.
		 * @return boolean
		 *  true if the `$needle` is in the `$haystack`, false otherwise.
		 */
		public static function in_iarray($needle, array $haystack){
			foreach($haystack as $key => $value){
				if(strcasecmp($value, $needle) == 0) return true;
			}
			return false;
		}

		/**
		 * Filter the input array for duplicates, treating each element in the array
		 * as a string and comparing them using a case insensitive comparison function.
		 *
		 * @param array $array
		 *  the array to filter.
		 * @return array
		 *  a new array containing only the unique elements of the input array.
		 */
		public static function array_iunique(array $array){
			$tmp = array();
			foreach($array as $key => $value){
				if(!self::in_iarray($value, $tmp)){
					$tmp[$key] = $value;
				}
			}
			return $tmp;
		}

		/**
		 * Function recursively apply a function to an array's values.
		 * This will not touch the keys, just the values.
		 *
		 * @since Symphony 2.2
		 * @param string $function
		 * @param array $array
		 * @return array
		 *  a new array with all the values passed through the given `$function`
		 */
		public static function array_map_recursive($function, array $array) {
			$tmp = array();
			foreach($array as $key => $value) {
				if(is_array($value)) {
					$tmp[$key] = self::array_map_recursive($function, $value);
				}
				else {
					$tmp[$key] = call_user_func($function, $value);
				}
			}
			return $tmp;
		}

		/**
		 * Convert an array into an XML fragment and append it to an existing
		 * XML element. Any arrays contained as elements in the input array will
		 * also be recursively formatted and appended to the input XML fragment.
		 * The input XML element will be modified as a result of calling this.
		 *
		 * @param XMLElement $parent
		 *  the XML element to append the formatted array data to.
		 * @param array $data
		 *  the array to format and append to the XML fragment.
		 * @param boolean $validate
		 *  true if the formatted array data should be validated as it is
		 *  constructed, false otherwise.
		 */
		public static function array_to_xml(XMLElement $parent, array $data, $validate=false) {
			foreach ($data as $element_name => $value) {
				if (empty($value)) continue;

				if (is_int($element_name)) {
					$child = new XMLElement('item');
					$child->setAttribute('index', $element_name + 1);
				}

				else {
					$child = new XMLElement($element_name, null, array(), true);
				}

				if(is_array($value)) {
					self::array_to_xml($child, $value);
					if($child->getNumberOfChildren() == 0) continue;
				}

				else if($validate == true && !self::validateXML(self::sanitize($value), $errors, false, new XSLTProcess)) {
					continue;
				}

				else {
					$child->setValue(self::sanitize($value));
				}

				$parent->appendChild($child);
			}
		}

		/**
		 * Create a file at the input path with the (optional) input permissions
		 * with the input content. This function will ignore errors in opening,
		 * writing, closing and changing the permissions of the resulting file.
		 * If opening or writing the file fail then this will return false.
		 *
		 * @param string $file
		 *  the path of the file to write.
		 * @param mixed $data
		 *  the data to write to the file.
		 * @param integer|null $perm (optional)
		 *  the permissions as an octal number to set set on the resulting file.
		 *  this defaults to 0644 (if omitted or set to null)
		 * @param string $mode (optional)
		 * the mode that the file should be opened with, defaults to 'w'. See modes
		 * at http://php.net/manual/en/function.fopen.php
		 * @return boolean
		 *  true if the file is successfully opened, written to, closed and has the
		 *  required permissions set. false, otherwise.
		 */
		public static function writeFile($file, $data, $perm = 0644, $mode = 'w'){
			if(
				(!is_writable(dirname($file)) || !is_readable(dirname($file))) // Folder
				|| (!is_readable($file) || !is_writable($file)) // File
			) {
				return false;
			}

			if(!$handle = fopen($file, $mode)) {
				return false;
			}<commit_msg>Fix General::writeFile logic to handle new files again. Closes #1543<commit_after><?php
	/**
	 * @package toolkit
	 */
	/**
	 * General is a utility class that offers a number miscellaneous of
	 * functions that are used throughout Symphony.
	 */
	Class General{

		/**
		 * Convert any special characters into their entity equivalents. Since
		 * Symphony 2.3, this function assumes UTF-8 and will not double
		 * encode strings.
		 *
		 * @param string $source
		 *  a string to operate on.
		 * @return string
		 *  the encoded version of the string.
		 */
		public static function sanitize($source) {
			$source = htmlspecialchars($source, ENT_COMPAT, 'UTF-8', false);

			return $source;
		}

		/**
		 * Revert any html entities to their character equivalents.
		 *
		 * @param string $str
		 *  a string to operate on
		 * @return string
		 *  the decoded version of the string
		 */
		public static function reverse_sanitize($str){
		   return htmlspecialchars_decode($str, ENT_COMPAT);
		}

		/**
		 * Validate a string against a set of regular expressions.
		 *
		 * @param array|string $string
		 *  string to operate on
		 * @param array|string $rule
		 *  a single rule or array of rules
		 * @return boolean
		 *  false if any of the rules in $rule do not match any of the strings in
		 *  `$string`, return true otherwise.
		 */
		public static function validateString($string, $rule){

			if(!is_array($rule) && $rule == '') return true;
			if(!is_array($string) && $string == '') return true;

			if(!is_array($rule)) $rule = array($rule);
			if(!is_array($string)) $string = array($string);

			foreach($rule as $r){
				foreach($string as $s){
					if(!preg_match($r, $s)) return false;
				}
			}
			return true;
		}

		/**
		 * Replace the tabs with spaces in the input string.
		 *
		 * @param string $string
		 *  the string in which to replace the tabs with spaces.
		 * @param integer $spaces (optional)
		 *  the number of spaces to replace each tab with. This argument is optional
		 *  with a default of 4.
		 * @return string
		 *  the resulting string.
		 */
		public static function tabsToSpaces($string, $spaces=4){
			return str_replace("\t", str_pad(NULL, $spaces), $string);
		}

		/**
		 * Checks an xml document for well-formedness.
		 *
		 * @param string $data
		 *  filename, xml document as a string, or arbitrary string
		 * @param pointer &$errors
		 *  pointer to an array which will contain any validation errors
		 * @param boolean $isFile (optional)
		 *  if this is true, the method will attempt to read from a file, `$data`
		 *  instead.
		 * @param mixed $xsltProcessor (optional)
		 *  if set, the validation will be done using this XSLT processor rather
		 *  than the built in XML parser. the default is null.
		 * @param string $encoding (optional)
		 *  if no XML header is expected, than this should be set to match the
		 *  encoding of the XML
		 * @return boolean
		 *  true if there are no errors in validating the XML, false otherwise.
		 */
		public static function validateXML($data, &$errors, $isFile=true, $xsltProcessor=NULL, $encoding='UTF-8') {
			$_parser = null;
			$_data = null;
			$_vals = array();
			$_index = array();

			$_data = ($isFile) ? file_get_contents($data) : $data;

			$_data = preg_replace('/<!DOCTYPE[-.:"\'\/\\w\\s]+>/', NULL, $_data);

			if(strpos($_data, '<?xml') === false){
				$_data = '<?xml version="1.0" encoding="'.$encoding.'"?><rootelement>'.$_data.'</rootelement>';
			}

			if(is_object($xsltProcessor)){

				$xsl = '<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">

				<xsl:template match="/"></xsl:template>

				</xsl:stylesheet>';

				$xsltProcessor->process($_data, $xsl, array());

				if($xsltProcessor->isErrors()) {
					$errors = $xsltProcessor->getError(true);
					return false;
				}

			}
			else{

				$_parser = xml_parser_create();
				xml_parser_set_option($_parser, XML_OPTION_SKIP_WHITE, 0);
				xml_parser_set_option($_parser, XML_OPTION_CASE_FOLDING, 0);

				if(!xml_parse($_parser, $_data)){
					$errors = array('error' => xml_get_error_code($_parser) . ': ' . xml_error_string(xml_get_error_code($_parser)),
									'col' => xml_get_current_column_number($_parser),
									'line' => (xml_get_current_line_number($_parser) - 2));
					return false;
				}

				xml_parser_free($_parser);
			}

			return true;

		}

		/**
		 * Check that a string is a valid URL.
		 *
		 * @param string $url
		 *  string to operate on
		 * @return string
		 *  a blank string or a valid URL
		 */
		public static function validateURL($url = null){
			if(is_null($url) || $url == '') return $url;

			if(!preg_match('#^http[s]?:\/\/#i', $url)){
				$url = 'http://' . $url;
			}

			include(TOOLKIT . '/util.validators.php');
			if(!preg_match($validators['URI'], $url)){
				$url = '';
			}

			return $url;
		}

		/**
		 * Strip any slashes from all array values.
		 *
		 * @param array &$arr
		 *  Pointer to an array to operate on. Can be multi-dimensional.
		 */
		public static function cleanArray(array &$arr) {
			foreach($arr as $k => $v){
				if(is_array($v))
					self::cleanArray($arr[$k]);
				else
					$arr[$k] = stripslashes($v);
			}
		}

		/**
		 * Flatten the input array. Any elements of the input array that are
		 * themselves arrays will be removed and the contents of the removed array
		 * inserted in its place. The keys for the inserted values will be the
		 * concatenation of the keys in the original arrays in which it was embedded.
		 * The elements of the path are separated by periods (.). For example,
		 * given the following nested array structure:
		 * `
		 * array(1 =>
		 *  		array('key' => 'value'),
		 *  	2 =>
		 *  		array('key2' => 'value2', 'key3' => 'value3')
		 *  	)
		 * `
		 * will flatten to:
		 * `array('1.key' => 'value', '2.key2' => 'value2', '2.key3' => 'value3')`
		 *
		 * @param array &$source
		 *  The array to flatten, passed by reference
		 * @param array &$output (optional)
		 *  The array in which to store the flattened input, passed by reference.
		 *  if this is not provided then a new array will be created.
		 * @param string $path (optional)
		 *  the current prefix of the keys to insert into the output array. this
		 *  defaults to null.
		 */
		public static function flattenArray(array &$source, &$output = null, $path = null) {
			if (is_null($output)) $output = array();

			foreach ($source as $key => $value) {
				if (is_int($key)) $key = (string)($key + 1);
				if (!is_null($path)) $key = $path . '.' . (string)$key;

				if (is_array($value)) self::flattenArray($value, $output, $key);
				else $output[$key] = $value;
			}

			$source = $output;
		}

		/**
		 * Flatten the input array. Any elements of the input array that are
		 * themselves arrays will be removed and the contents of the removed array
		 * inserted in its place. The keys for the inserted values will be the
		 * concatenation of the keys in the original arrays in which it was embedded.
		 * The elements of the path are separated by colons (:). For example, given
		 * the following nested array structure:
		 * `
		 * array(1 =>
		 *  		array('key' => 'value'),
		 *  	2 =>
		 *  		array('key2' => 'value2', 'key3' => 'value3')
		 *  	)
		 * `
		 * will flatten to:
		 * `array('1:key' => 'value', '2:key2' => 'value2', '2:key3' => 'value3')`
		 *
		 *
		 * @param array &$output
		 *  The array in which to store the flattened input, passed by reference.
		 * @param array &$source
		 *  The array to flatten, passed by reference
		 * @param string $path
		 *  the current prefix of the keys to insert into the output array.
		 */
		protected static function flattenArraySub(array &$output, array &$source, $path) {
			foreach ($source as $key => $value) {
				$key = $path . ':' . $key;

				if (is_array($value)) self::flattenArraySub($output, $value, $key);
				else $output[$key] = $value;
			}
		}

		/**
		 * Allows you to send emails. It initializes the core email class.
		 *
		 * @deprecated Since Symphony 2.2
		 * @param string $to_email
		 *  email of the recipient
		 * @param string $from_email
		 *  the from email address. This is usually your email
		 * @param string $from_name
		 *  the name of the sender
		 * @param string $subject
		 *  subject of the email
		 * @param string $message
		 *  contents of the email
		 * @param array $additional_headers
		 *  an array containing additional email headers. This will NOT work
		 *  for Content-Type header fields which will be added/overwritten by
		 *  the email gateways.)
		 * @return boolean
		 *  true on success
		 */
		public static function sendEmail($to_email, $from_email, $from_name, $subject, $message, array $additional_headers = array()) {

			try{
				$email = Email::create();

				if (!empty($additional_headers)) {
					foreach ($additional_headers as $name => $body) {
						$email->appendHeaderField($name, $body);
					}
				}
				$email->sender_name = $from_name;
				$email->sender_email_address = $from_email;

				$email->recipients = $email->setRecipients($to_email);
				$email->text_plain = $message;
				$email->subject = $subject;

				return $email->send();
			}
			catch(EmailGatewayException $e){
				throw new SymphonyErrorPage('Error sending email. ' . $e->getMessage());
			}
			catch(EmailException $e){
				throw new SymphonyErrorPage('Error sending email. ' . $e->getMessage());
			}
		}

		/**
		 * Given a string, this will clean it for use as a Symphony handle. Preserves multi-byte characters.
		 *
		 * @since Symphony 2.2.1
		 * @param string $string
		 *  String to be cleaned up
		 * @param int $max_length
		 *  The maximum number of characters in the handle
		 * @param string $delim
		 *  All non-valid characters will be replaced with this
		 * @param boolean $uriencode
		 *  Force the resultant string to be uri encoded making it safe for URLs
		 * @param array $additional_rule_set
		 *  An array of REGEX patterns that should be applied to the `$string`. This
		 *  occurs after the string has been trimmed and joined with the `$delim`
		 * @return string
		 *  Returns resultant handle
		 */
		public static function createHandle($string, $max_length=255, $delim='-', $uriencode=false, $additional_rule_set=NULL) {

			$max_length = intval($max_length);

			// Strip out any tag
			$string = strip_tags($string);

			// Remove punctuation
			$string = preg_replace('/[\\.\'"]+/', NULL, $string);

			// Trim it
			if($max_length > 0) $string = General::limitWords($string, $max_length);

			// Replace spaces (tab, newline etc) with the delimiter
			$string = preg_replace('/[\s]+/', $delim, $string);

			// Find all legal characters
			preg_match_all('/[^<>?@:!-\/\[-`;‘’…]+/u', $string, $matches);

			// Join only legal character with the $delim
			$string = implode($delim, $matches[0]);

			// Allow for custom rules
			if(is_array($additional_rule_set) && !empty($additional_rule_set)) {
				foreach($additional_rule_set as $rule => $replacement) $string = preg_replace($rule, $replacement, $string);
			}

			// Remove leading or trailing delim characters
			$string = trim($string, $delim);

			// Encode it for URI use
			if($uriencode) $string = urlencode($string);

			// Make it lowercase
			$string = strtolower($string);

			return $string;

		}

		/**
		 * Given a string, this will clean it for use as a filename. Preserves multi-byte characters.
		 *
		 * @since Symphony 2.2.1
		 * @param string $string
		 *  String to be cleaned up
		 * @param string $delim
		 *  All non-valid characters will be replaced with this
		 * @return string
		 *  Returns created filename
		 */
		public static function createFilename($string, $delim='-') {

			// Strip out any tag
			$string = strip_tags($string);

			// Find all legal characters
			$count = preg_match_all('/[\p{L}\w:;.,+=~]+/u', $string, $matches);
			if($count <= 0 || $count == false) {
				preg_match_all('/[\w:;.,+=~]+/', $string, $matches);
			}

			// Join only legal character with the $delim
			$string = implode($delim, $matches[0]);

			// Remove leading or trailing delim characters
			$string = trim($string, $delim);

			// Make it lowercase
			$string = strtolower($string);

			return $string;

		}

		/**
		 * Extract the first `$val` characters of the input string. If `$val`
		 * is larger than the length of the input string then the original
		 * input string is returned.
		 *
		 * @param string $str
		 *  the string to operate on
		 * @param integer $val
		 *  the number to compare lengths with
		 * @return string|boolean
		 *  the resulting string or false on failure.
		 */
		public static function substrmin($str, $val){
			return(substr($str, 0, min(strlen($str), $val)));
		}

		/**
		 * Extract the first `$val` characters of the input string. If
		 * `$val` is larger than the length of the input string then
		 * the original input string is returned
		 *
		 * @param string $str
		 *  the string to operate on
		 * @param integer $val
		 *  the number to compare lengths with
		 * @return string|boolean
		 *  the resulting string or false on failure.
		 */
		public static function substrmax($str, $val){
			return(substr($str, 0, max(strlen($str), $val)));
		}

		/**
		 * Extract the last `$num` characters from a string.
		 *
		 * @param string $str
		 *  the string to extract the characters from.
		 * @param integer $num
		 *  the number of characters to extract.
		 * @return string|boolean
		 *  a string containing the last `$num` characters of the
		 *  input string, or false on failure.
		 */
		public static function right($str, $num){
			$str = substr($str, strlen($str)-$num,  $num);
			return $str;
		}

		/**
		 * Extract the first `$num` characters from a string.
		 *
		 * @param string $str
		 *  the string to extract the characters from.
		 * @param integer $num
		 *  the number of characters to extract.
		 * @return string|boolean
		 *  a string containing the last `$num` characters of the
		 *  input string, or false on failure.
		 */
		public static function left($str, $num){
			$str = substr($str, 0, $num);
			return $str;
		}

		/**
		 * Create all the directories as specified by the input path. If the current
		 * directory already exists, this function will return true.
		 *
		 * @param string $path
		 *  the path containing the directories to create.
		 * @param integer $mode (optional)
		 *  the permissions (in octal) of the directories to create. Defaults to 0755
		 * @return boolean
		 */
		public static function realiseDirectory($path, $mode=0755){
			if(is_dir($path)) return true;

			$current_umask = umask(0);
			$success = mkdir($path, intval($mode, 8), true);
			umask($current_umask);

			return $success;
		}

		/**
		 * Recursively deletes all files and directories given a directory. This
		 * function has two path. This function optionally takes a `$silent` parameter,
		 * which when `false` will throw an `Exception` if there is an error deleting a file
		 * or folder.
		 *
		 * @since Symphony 2.3
		 * @param string $dir
		 *  the path of the directory to delete
		 * @param boolean $silent (optional)
		 *  true if an exception should be raised if an error occurs, false
		 *  otherwise. this defaults to true.
		 * @return boolean
		 */
		public static function deleteDirectory($dir, $silent = true) {
			try {
				if (!file_exists($dir)) return true;

				if (!is_dir($dir)) return unlink($dir);

				foreach (scandir($dir) as $item) {
					if ($item == '.' || $item == '..') continue;

					if (!self::deleteDirectory($dir.DIRECTORY_SEPARATOR.$item)) return false;
				}

				return rmdir($dir);
			}
			catch(Exception $ex) {
				if($silent === false){
					throw new Exception(__('Unable to remove - %s', array($dir)));
				}

				return false;
			}
		}

		/**
		 * Search a multi-dimensional array for a value.
		 *
		 * @param mixed $needle
		 *  the value to search for.
		 * @param array $haystack
		 *  the multi-dimensional array to search.
		 * @return boolean
		 *  true if `$needle` is found in `$haystack`.
		 *  true if `$needle` == `$haystack`.
		 *  true if `$needle` is found in any of the arrays contained within `$haystack`.
		 *  false otherwise.
		 */
		public static function in_array_multi($needle, $haystack){

			if($needle == $haystack) return true;

			if(is_array($haystack)){
				foreach($haystack as $key => $val){

					if(is_array($val)){
						if(self::in_array_multi($needle, $val)) return true;
					}

					elseif(!strcmp($needle, $key) || !strcmp($needle, $val)){
						return true;
					}
				}
			}

			return false;
		}

		/**
		 * Search an array for multiple values.
		 *
		 * @param array $needles
		 *  the values to search the `$haystack` for.
		 * @param array $haystack
		 *  the in which to search for the `$needles`
		 * @return boolean
		 *  true if any of the `$needles` are in `$haystack`,
		 *  false otherwise.
		 */
		public static function in_array_all($needles, $haystack){

			foreach($needles as $n){
				if(!in_array($n, $haystack)) return false;
			}

			return true;
		}

		/**
		 * Transform a multi-dimensional array to a flat array. The input array
		 * is expected to conform to the structure of the `$_FILES` variable.
		 *
		 * @param array $filedata
		 *  the raw `$_FILES` data structured array
		 * @return array
		 *  the flattened array.
		 */
		public static function processFilePostData($filedata){

			$result = array();

			foreach($filedata as $key => $data){
				foreach($data as $handle => $value){
					if(is_array($value)){
						foreach($value as $index => $pair){

							if(!is_array($result[$handle][$index])) $result[$handle][$index] = array();

							if(!is_array($pair)) $result[$handle][$index][] = $pair;
							else $result[$handle][$index][array_pop(array_keys($pair))][$key] = array_pop(array_values($pair));
						}
					}

					else $result[$handle][$key] = $value;
				}
			}

			return $result;
		}

		/**
		 * Merge `$_POST` with `$_FILES` to produce a flat array of the contents
		 * of both. If there is no merge_file_post_data function defined then
		 * such a function is created. This is necessary to overcome PHP's ability
		 * to handle forms. This overcomes PHP's convoluted `$_FILES` structure
		 * to make it simpler to access `multi-part/formdata`.
		 *
		 * @return array
		 *  a flat array containing the flattened contents of both `$_POST` and
		 *  `$_FILES`.
		 */
		public static function getPostData() {
			if (!function_exists('merge_file_post_data')) {
				function merge_file_post_data($type, array $file, &$post) {
					foreach ($file as $key => $value) {
						if (!isset($post[$key])) $post[$key] = array();
						if (is_array($value)) merge_file_post_data($type, $value, $post[$key]);
						else $post[$key][$type] = $value;
					}
				}
			}

			$files = array(
				'name'		=> array(),
				'type'		=> array(),
				'tmp_name'	=> array(),
				'error'		=> array(),
				'size'		=> array()
			);
			$post = $_POST;

			if(is_array($_FILES) && !empty($_FILES)){
				foreach ($_FILES as $key_a => $data_a) {
					if(!is_array($data_a)) continue;
					foreach ($data_a as $key_b => $data_b) {
						$files[$key_b][$key_a] = $data_b;
					}
				}
			}

			foreach ($files as $type => $data) {
				merge_file_post_data($type, $data, $post);
			}

			return $post;
		}

		/**
		 * Find the next available index in an array. Works best with numeric keys.
		 * The next available index is the minimum integer such that the array does
		 * not have a mapping for that index. Uses the increment operator on the
		 * index type of the input array, whatever that may do.
		 *
		 * @param array $array
		 *  the array to find the next index for.
		 * @param mixed $seed (optional)
		 *  the object with which the search for an empty index is initialized. this
		 *  defaults to null.
		 * @return integer
		 *  the minimum empty index into the input array.
		 */
		public static function array_find_available_index($array, $seed=NULL){

			if(!is_null($seed)) $index = $seed;
			else{
				$keys = array_keys($array);
				sort($keys);
				$index = array_pop($keys);
			}

			if(isset($array[$index])){
				do{
					$index++;
				}while(isset($array[$index]));
			}

			return $index;
		}

		/**
		 * Filter the duplicate values from an array into a new array, optionally
		 * ignoring the case of the values (assuming they are strings?). A new array
		 * is returned, the input array is left unchanged.
		 *
		 * @param array $array
		 *  the array to filter.
		 * @param boolean $ignore_case
		 *  true if the case of the values in the array should be ignored, false otherwise.
		 * @return
		 *  a new array containing only the unique elements of the input array.
		 */
		public static function array_remove_duplicates(array $array, $ignore_case=false){
			return ($ignore_case == true ? self::array_iunique($array) : array_unique($array));
		}

		/**
		 * Test whether a value is in an array based on string comparison, ignoring
		 * the case of the values.
		 *
		 * @param mixed $needle
		 *  the object to search the array for.
		 * @param array $haystack
		 *  the array to search for the `$needle`.
		 * @return boolean
		 *  true if the `$needle` is in the `$haystack`, false otherwise.
		 */
		public static function in_iarray($needle, array $haystack){
			foreach($haystack as $key => $value){
				if(strcasecmp($value, $needle) == 0) return true;
			}
			return false;
		}

		/**
		 * Filter the input array for duplicates, treating each element in the array
		 * as a string and comparing them using a case insensitive comparison function.
		 *
		 * @param array $array
		 *  the array to filter.
		 * @return array
		 *  a new array containing only the unique elements of the input array.
		 */
		public static function array_iunique(array $array){
			$tmp = array();
			foreach($array as $key => $value){
				if(!self::in_iarray($value, $tmp)){
					$tmp[$key] = $value;
				}
			}
			return $tmp;
		}

		/**
		 * Function recursively apply a function to an array's values.
		 * This will not touch the keys, just the values.
		 *
		 * @since Symphony 2.2
		 * @param string $function
		 * @param array $array
		 * @return array
		 *  a new array with all the values passed through the given `$function`
		 */
		public static function array_map_recursive($function, array $array) {
			$tmp = array();
			foreach($array as $key => $value) {
				if(is_array($value)) {
					$tmp[$key] = self::array_map_recursive($function, $value);
				}
				else {
					$tmp[$key] = call_user_func($function, $value);
				}
			}
			return $tmp;
		}

		/**
		 * Convert an array into an XML fragment and append it to an existing
		 * XML element. Any arrays contained as elements in the input array will
		 * also be recursively formatted and appended to the input XML fragment.
		 * The input XML element will be modified as a result of calling this.
		 *
		 * @param XMLElement $parent
		 *  the XML element to append the formatted array data to.
		 * @param array $data
		 *  the array to format and append to the XML fragment.
		 * @param boolean $validate
		 *  true if the formatted array data should be validated as it is
		 *  constructed, false otherwise.
		 */
		public static function array_to_xml(XMLElement $parent, array $data, $validate=false) {
			foreach ($data as $element_name => $value) {
				if (empty($value)) continue;

				if (is_int($element_name)) {
					$child = new XMLElement('item');
					$child->setAttribute('index', $element_name + 1);
				}

				else {
					$child = new XMLElement($element_name, null, array(), true);
				}

				if(is_array($value)) {
					self::array_to_xml($child, $value);
					if($child->getNumberOfChildren() == 0) continue;
				}

				else if($validate == true && !self::validateXML(self::sanitize($value), $errors, false, new XSLTProcess)) {
					continue;
				}

				else {
					$child->setValue(self::sanitize($value));
				}

				$parent->appendChild($child);
			}
		}

		/**
		 * Create a file at the input path with the (optional) input permissions
		 * with the input content. This function will ignore errors in opening,
		 * writing, closing and changing the permissions of the resulting file.
		 * If opening or writing the file fail then this will return false.
		 *
		 * @param string $file
		 *  the path of the file to write.
		 * @param mixed $data
		 *  the data to write to the file.
		 * @param integer|null $perm (optional)
		 *  the permissions as an octal number to set set on the resulting file.
		 *  this defaults to 0644 (if omitted or set to null)
		 * @param string $mode (optional)
		 * the mode that the file should be opened with, defaults to 'w'. See modes
		 * at http://php.net/manual/en/function.fopen.php
		 * @return boolean
		 *  true if the file is successfully opened, written to, closed and has the
		 *  required permissions set. false, otherwise.
		 */
		public static function writeFile($file, $data, $perm = 0644, $mode = 'w'){
			if(
				(!is_writable(dirname($file)) || !is_readable(dirname($file))) // Folder
				|| (file_exists($file) && (!is_readable($file) || !is_writable($file))) // File
			) {
				return false;
			}

			if(!$handle = fopen($file, $mode)) {
				return false;
			}
{'fraction_non_alphanumeric': 0.12995966700443068, 'fraction_numerical': 0.003616061671733127, 'mean_word_length': 2.999364322606277, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 2, '>': 139, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 34}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '353053', 'n_tokens_mistral': 18523, 'n_tokens_neox': 14833, 'n_words': 5382}