<commit_before>package org.mangui.hls.utils
{
	public class StringUtil
	{
		/**
		 * Returns <code>true</code> if the specified string is
		 * a single space, tab, carriage return, newline, or formfeed character.
		 *
		 * @param str The String that is is being queried. 
		 *
		 * @return <code>true</code> if the specified string is
		 * a single space, tab, carriage return, newline, or formfeed character.
		 */
		public static function isWhitespace(character:String):Boolean
		{
			switch (character)
			{
				case " ":
				case "\t":
				case "\r":
				case "\n":
				case "\f":
				case "\u00A0": // non breaking space
				case "\u2028": // line seperator
				case "\u2029": // paragraph seperator
				case "\u3000": // ideographic space
					return true;
			}
			
			return false;
		}
		
		/**
		 * Removes all whitespace characters from the beginning and end
		 * of the specified string.
		 *
		 * @param str The String whose whitespace should be trimmed. 
		 *
		 * @return Updated String where whitespace was removed from the 
		 * beginning and end. 
		 */
		public static function trim(str:String):String
		{
			if (str == null) return '';
			
			var startIndex:int = 0;
			while (isWhitespace(str.charAt(startIndex)))
				++startIndex;
			
			var endIndex:int = str.length - 1;
			while (isWhitespace(str.charAt(endIndex)))
				--endIndex;
			
			if (endIndex >= startIndex)
				return str.slice(startIndex, endIndex + 1);
			else
				return "";
		}
		
		/**
		 * Splits a String into an Vector of nicely trimmed strings, where each
		 * item represents a single line of the original String data
		 * 
		 * @param	str		The String to be split into an Array
		 * @return			Vector of type String 
		 */
		public static function toLines(str:String):Vector.<String>
		{
			var lines:Array = (str || "").split("\n");
			var i:uint;
			var length:uint = lines.length;
			
			for (i=0; i<length; ++i)
			{
				lines[i] = trim(lines[i]);
			}
			
			return Vector.<String>(lines);
		}
		
		/**
		 * Converts strings containing Windows (CR-LF), MacOS (CR) and other 
		 * non-standard line breaks (LF-CR) into strings using only Linux-style
		 * line breaks (LF).
		 * 
		 * @param	str		String containing non-Linux line breaks
		 * @returns			String containly only Linux-style line breaks 
		 */
		public static function toLF(str:String):String
		{
			return (str || "").replace(/\r\n|\n\r|\r/g, "\n");
		}
	}
}<commit_msg>Ensure all line breaks are LF before splitting into lines<commit_after>package org.mangui.hls.utils
{
	public class StringUtil
	{
		/**
		 * Returns <code>true</code> if the specified string is
		 * a single space, tab, carriage return, newline, or formfeed character.
		 *
		 * @param str The String that is is being queried. 
		 *
		 * @return <code>true</code> if the specified string is
		 * a single space, tab, carriage return, newline, or formfeed character.
		 */
		public static function isWhitespace(character:String):Boolean
		{
			switch (character)
			{
				case " ":
				case "\t":
				case "\r":
				case "\n":
				case "\f":
				case "\u00A0": // non breaking space
				case "\u2028": // line seperator
				case "\u2029": // paragraph seperator
				case "\u3000": // ideographic space
					return true;
			}
			
			return false;
		}
		
		/**
		 * Removes all whitespace characters from the beginning and end
		 * of the specified string.
		 *
		 * @param str The String whose whitespace should be trimmed. 
		 *
		 * @return Updated String where whitespace was removed from the 
		 * beginning and end. 
		 */
		public static function trim(str:String):String
		{
			if (str == null) return '';
			
			var startIndex:int = 0;
			while (isWhitespace(str.charAt(startIndex)))
				++startIndex;
			
			var endIndex:int = str.length - 1;
			while (isWhitespace(str.charAt(endIndex)))
				--endIndex;
			
			if (endIndex >= startIndex)
				return str.slice(startIndex, endIndex + 1);
			else
				return "";
		}
		
		/**
		 * Splits a String into an Vector of nicely trimmed strings, where each
		 * item represents a single line of the original String data
		 * 
		 * @param	str		The String to be split into an Array
		 * @return			Vector of type String 
		 */
		public static function toLines(str:String):Vector.<String>
		{
			var lines:Array = toLF(str).split("\n");
			var i:uint;
			var length:uint = lines.length;
			
			for (i=0; i<length; ++i)
			{
				lines[i] = trim(lines[i]);
			}
			
			return Vector.<String>(lines);
		}
		
		/**
		 * Converts strings containing Windows (CR-LF), MacOS (CR) and other 
		 * non-standard line breaks (LF-CR) into strings using only Linux-style
		 * line breaks (LF).
		 * 
		 * @param	str		String containing non-Linux line breaks
		 * @returns			String containly only Linux-style line breaks 
		 */
		public static function toLF(str:String):String
		{
			return (str || "").replace(/\r\n|\n\r|\r/g, "\n");
		}
	}
}
{'fraction_non_alphanumeric': 0.11565696302124312, 'fraction_numerical': 0.007474429583005508, 'mean_word_length': 2.3278795811518322, 'pattern_counts': {'":': 18, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1198964', 'n_tokens_mistral': 1993, 'n_tokens_neox': 1506, 'n_words': 506}