<commit_before>dojo.provide("dojox.string.BidiEngine");

(function(){

dojo.declare("dojox.string.BidiEngine", null, {
	// summary:
	//		This class provides a bidi transformation engine, i.e.
	//		functions for reordering and shaping bidi text.
	// description:
	//		Bidi stands for support for languages with a bidirectional script. 
	//
	//		Usually Unicode Bidi Algorithm used by OS platform (and web browsers) is capable of properly transforming
	//		Bidi text and as a result it is adequately displayed on the screen. However, in some situations, 
	//		Unicode Bidi Algorithm is not invoked or is not properly applied. This may occur in situation in which software
	//		responsible for rendering the text is not leveraging Unicode Bidi Algorithm implemented by OS (e.g. dojox.GFX renderers).
	//
	//		Bidi engine provided in this class implements Unicode Bidi Algorithm as specified at:
	//			http://www.unicode.org/reports/tr9/. 
	//
	//		For more information on basic Bidi concepts please read following article:
	//			"Bidirectional script support - A primer" available from:
	//				http://www.ibm.com/developerworks/websphere/library/techarticles/bidi/bidigen.html
	//
	//		As of February 2011, Bidi engine has following limitations:
	//			1. No support for following numeric shaping options: 
	//				H - Hindi, 
	//				C - Contextual, 
	//				N - Nominal.
	//			2. No support for following shaping options: 
	//				I - Initial shaping, 
	//				M - Middle shaping, 
	//				F - Final shaping, 
	//				B - Isolated shaping.
	//			3. No support for source-to-target or/and target-to-source maps.
	//			4. No support for LRE/RLE/LRO/RLO/PDF (they are handled like neutrals).
	//			5. No support for Windows compatibility.
	//			6. No support for  insert/remove marks.
	//			7. No support for code pages (currently only UTF-8 is supported. Ideally we should convert from any code page to UTF-8).
	
	bidiTransform: function (/*String*/text, /*String*/formatIn, /*String*/formatOut){
		// summary:
		//		Central public API for Bidi engine. Transforms the text according to formatIn, formatOut parameters.
		//		If formatIn or formatOut parametrs are not valid throws an exception.
		// inputText:
		//		Input text subject to application of Bidi transformation.
		// formatIn:
		//		Input Bidi layout in which inputText is passed to the function.
		// formatOut:
		//		Output Bidi layout to which inputText should be transformed.
		// description:
		//		Both formatIn and formatOut parameters are 5 letters long strings. 
		//		For example - "ILYNN". Each letter is associated with specific attribute of Bidi layout. 
		//		Possible and default values for each one of the letters are provided below:
		//
		//		First letter:
		//			Letter position/index:	
		//				1
		//			Letter meaning:			
		//				Ordering Schema.
		//			Possible values:
		//				I - Implicit (Logical).
		//				V - Visual.
		//			Default value:
		//				I
		//
		//		Second letter:
		//			Letter position/index:	
		//				2
		//			Letter meaning:			
		//				Orientation.
		//			Possible values:
		//				L - Left To Right.
		//				R - Right To Left.
		//				C - Contextual Left to Right.
		//				D - Contextual Right to Left.
		//			Default value:
		//				L		
		//
		//		Third letter:
		//			Letter position/index:	
		//				3
		//			Letter meaning:			
		//				Symmetric Swapping.
		//			Possible values:
		//				Y - Symmetric swapping is on.
		//				N - Symmetric swapping is off.
		//			Default value:
		//				Y		
		//
		//		Fourth letter:
		//			Letter position/index:	
		//				4
		//			Letter meaning:			
		//				Shaping.
		//			Possible values:
		//				S - Text is shaped.
		//				N - Text is not shaped.
		//			Default value:
		//				N				
		//
		//		Fifth letter:
		//			Letter position/index:	
		//				5
		//			Letter meaning:			
		//				Numeric Shaping.
		//			Possible values:
		//				N - Nominal.
		//			Default value:
		//				N				
		//
		//		The output of this function is original text (passed via first argument) transformed from input Bidi layout (second argument)
		//		to output Bidi layout (last argument). 
		//
		//		Sample call:
		//	|	mytext = bidiTransform("HELLO WORLD", "ILYNN", "VLYNN");
		//		In this case, "HELLO WORLD" text is transformed from Logical - LTR to Visual - LTR Bidi layout with 
		//		default values for symmetric swapping (Yes), shaping (Not shaped) and numeric shaping (Nominal).
		// returns: /*String*/ or throws an exception.
		//		Original text transformed from input Bidi layout (second argument)
		//		to output Bidi layout (last argument).
		//		Throws an exception if the bidi layout strings are not valid.
		// tags:
		//		public
		
		if(!text){
			return '';
		}
		if(!formatIn && !formatOut){
			return text;
		}

		// regex for format validation
		// Allowed values for format string are:
		// 1st letter- I, V
		// 2nd letter- L, R, C, D
		// 3rd letter- Y, N
		// 4th letter- S, N
		// 5th letter- N
		var validFormat = /^[(I|V)][(L|R|C|D)][(Y|N)][(S|N)][N]$/;
		if(!validFormat.test(formatIn) || !validFormat.test(formatOut)){
			throw new Error("dojox.string.BidiEngine: the bidi layout string is wrong!");
		}

		if(formatIn == formatOut){
			return text;
		}

		var orientIn = getOrientation(formatIn.charAt(1))
			, orientOut = getOrientation(formatOut.charAt(1))
			, os_in = (formatIn.charAt(0) == 'I') ? 'L' : formatIn.charAt(0)
			, os_out = (formatOut.charAt(0) == 'I') ? 'L' : formatOut.charAt(0)
			, inFormat = os_in + orientIn
			, outFormat = os_out + orientOut
			, swap = formatIn.charAt(2) + formatOut.charAt(2)
			;

		if(inFormat){
			bdx.defInFormat = inFormat;
		}
		if(outFormat){
			bdx.defOutFormat = outFormat;
		}
		if(swap){
			bdx.defSwap = swap;
		}
		
		var stage1_text = doBidiReorder(text, os_in + orientIn, os_out + orientOut, formatIn.charAt(2) + formatOut.charAt(2))
			, isRtl = false;

		if(formatIn.charAt(1) == 'R'){
			isRtl = true;
		}else if(formatIn.charAt(1) == 'C' || formatIn.charAt(1) == 'D'){
			isRtl = this.checkContextual(stage1_text);<commit_msg>Arabic shaping support. patch from <NAME>, IBM/CCLA\!strict Fixes #12583<commit_after>dojo.provide("dojox.string.BidiEngine");

(function(){

dojo.declare("dojox.string.BidiEngine", null, {
	// summary:
	//		This class provides a bidi transformation engine, i.e.
	//		functions for reordering and shaping bidi text.
	// description:
	//		Bidi stands for support for languages with a bidirectional script. 
	//
	//		Usually Unicode Bidi Algorithm used by OS platform (and web browsers) is capable of properly transforming
	//		Bidi text and as a result it is adequately displayed on the screen. However, in some situations, 
	//		Unicode Bidi Algorithm is not invoked or is not properly applied. This may occur in situation in which software
	//		responsible for rendering the text is not leveraging Unicode Bidi Algorithm implemented by OS (e.g. dojox.GFX renderers).
	//
	//		Bidi engine provided in this class implements Unicode Bidi Algorithm as specified at:
	//			http://www.unicode.org/reports/tr9/. 
	//
	//		For more information on basic Bidi concepts please read following article:
	//			"Bidirectional script support - A primer" available from:
	//				http://www.ibm.com/developerworks/websphere/library/techarticles/bidi/bidigen.html
	//
	//		As of February 2011, Bidi engine has following limitations:
	//			1. No support for following numeric shaping options: 
	//				H - Hindi, 
	//				C - Contextual, 
	//				N - Nominal.
	//			2. No support for following shaping options: 
	//				I - Initial shaping, 
	//				M - Middle shaping, 
	//				F - Final shaping, 
	//				B - Isolated shaping.
	//			3. No support for source-to-target or/and target-to-source maps.
	//			4. No support for LRE/RLE/LRO/RLO/PDF (they are handled like neutrals).
	//			5. No support for Windows compatibility.
	//			6. No support for  insert/remove marks.
	//			7. No support for code pages (currently only UTF-8 is supported. Ideally we should convert from any code page to UTF-8).
	
	bidiTransform: function (/*String*/text, /*String*/formatIn, /*String*/formatOut){
		// summary:
		//		Central public API for Bidi engine. Transforms the text according to formatIn, formatOut parameters.
		//		If formatIn or formatOut parametrs are not valid throws an exception.
		// inputText:
		//		Input text subject to application of Bidi transformation.
		// formatIn:
		//		Input Bidi layout in which inputText is passed to the function.
		// formatOut:
		//		Output Bidi layout to which inputText should be transformed.
		// description:
		//		Both formatIn and formatOut parameters are 5 letters long strings. 
		//		For example - "ILYNN". Each letter is associated with specific attribute of Bidi layout. 
		//		Possible and default values for each one of the letters are provided below:
		//
		//		First letter:
		//			Letter position/index:	
		//				1
		//			Letter meaning:			
		//				Ordering Schema.
		//			Possible values:
		//				I - Implicit (Logical).
		//				V - Visual.
		//			Default value:
		//				I
		//
		//		Second letter:
		//			Letter position/index:	
		//				2
		//			Letter meaning:			
		//				Orientation.
		//			Possible values:
		//				L - Left To Right.
		//				R - Right To Left.
		//				C - Contextual Left to Right.
		//				D - Contextual Right to Left.
		//			Default value:
		//				L		
		//
		//		Third letter:
		//			Letter position/index:	
		//				3
		//			Letter meaning:			
		//				Symmetric Swapping.
		//			Possible values:
		//				Y - Symmetric swapping is on.
		//				N - Symmetric swapping is off.
		//			Default value:
		//				Y		
		//
		//		Fourth letter:
		//			Letter position/index:	
		//				4
		//			Letter meaning:			
		//				Shaping.
		//			Possible values:
		//				S - Text is shaped.
		//				N - Text is not shaped.
		//			Default value:
		//				N				
		//
		//		Fifth letter:
		//			Letter position/index:	
		//				5
		//			Letter meaning:			
		//				Numeric Shaping.
		//			Possible values:
		//				N - Nominal.
		//			Default value:
		//				N				
		//
		//		The output of this function is original text (passed via first argument) transformed from input Bidi layout (second argument)
		//		to output Bidi layout (last argument). 
		//
		//		Sample call:
		//	|	mytext = bidiTransform("HELLO WORLD", "ILYNN", "VLYNN");
		//		In this case, "HELLO WORLD" text is transformed from Logical - LTR to Visual - LTR Bidi layout with 
		//		default values for symmetric swapping (Yes), shaping (Not shaped) and numeric shaping (Nominal).
		// returns: /*String*/ or throws an exception.
		//		Original text transformed from input Bidi layout (second argument)
		//		to output Bidi layout (last argument).
		//		Throws an exception if the bidi layout strings are not valid.
		// tags:
		//		public
		
		if(!text){
			return '';
		}
		if(!formatIn && !formatOut){
			return text;
		}

		// regex for format validation
		// Allowed values for format string are:
		// 1st letter- I, V
		// 2nd letter- L, R, C, D
		// 3rd letter- Y, N
		// 4th letter- S, N
		// 5th letter- N
		var validFormat = /^[(I|V)][(L|R|C|D)][(Y|N)][(S|N)][N]$/;
		if(!validFormat.test(formatIn) || !validFormat.test(formatOut)){
			throw new Error("dojox.string.BidiEngine: the bidi layout string is wrong!");
		}

		if(formatIn == formatOut){
			return text;
		}

		var orientIn = getOrientation(formatIn.charAt(1))
			, orientOut = getOrientation(formatOut.charAt(1))
			, os_in = (formatIn.charAt(0) == 'I') ? 'L' : formatIn.charAt(0)
			, os_out = (formatOut.charAt(0) == 'I') ? 'L' : formatOut.charAt(0)
			, inFormat = os_in + orientIn
			, outFormat = os_out + orientOut
			, swap = formatIn.charAt(2) + formatOut.charAt(2)
			;

		if(inFormat){
			bdx.defInFormat = inFormat;
		}
		if(outFormat){
			bdx.defOutFormat = outFormat;
		}
		if(swap){
			bdx.defSwap = swap;
		}
		
		var stage1_text = doBidiReorder(text, os_in + orientIn, os_out + orientOut, formatIn.charAt(2) + formatOut.charAt(2))
			, isRtl = false;

		if(formatOut.charAt(1) == 'R'){
			isRtl = true;
		}else if(formatOut.charAt(1) == 'C' || formatOut.charAt(1) == 'D'){
			isRtl = this.checkContextual(stage1_text);
{'fraction_non_alphanumeric': 0.12219286657859973, 'fraction_numerical': 0.0070178335535006605, 'mean_word_length': 2.3572616407982263, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '74445', 'n_tokens_mistral': 4835, 'n_tokens_neox': 4100, 'n_words': 1293}