<commit_before><commit_msg>add poc.xml with gb2312 encoding<commit_after><?xml version="1.0" encoding="gb2312" ?>

<cma>
	<entities>
		<!-- naming for these entiries are fixed -->
		<digit><![CDATA[0123456789]]></digit>
		<chardigit><![CDATA[һ߰˾šإئʮǧ]]></chardigit>
		<letter><![CDATA[QWERTYUIOPqwertyuiopASDFGHJKLasdfghjklZXCVBNMzxcvbnmѣףţңԣ٣գɣϣУӣģƣǣȣʣˣ̣ڣأã֣£Σͣ]]></letter>
		<punctuation><![CDATA[~!@#%^&*()` {}[]:"|;'\<>?,./.""''+-=_.ޣࡡۣݣܣ]]></punctuation>		
		<space><![CDATA[ ]]></space>
		<sentenceseparator><![CDATA[?!]]></sentenceseparator>
	</entities>

	<rules>
		<rule>
			<char><![CDATA[ǧ]]></char>
			<condition>
				<type>other</type>
				<nextchar><![CDATA[߿˷]]></nextchar>
			</condition>
		</rule>
		
		<rule>
			<char><![CDATA[]]></char>
			<condition>
				<type>other</type>
				<nextchar><![CDATA[ԲԪ]]></nextchar>
			</condition>
		</rule>

		<rule>
			<char><![CDATA[]]></char>
			<condition>
				<type>other</type>
				<nextchar><![CDATA[ȴ]]></nextchar>
			</condition>
		</rule>

		<rule>
			<char><![CDATA[]]></char>
			<condition>
				<type>other</type>
				<nextchar><![CDATA[ǮӲ]]></nextchar>
			</condition>
			<condition>
				<type>chardigit</type>
				<pretype>chardigit</pretype>
				<nextchar><![CDATA[֮]]></nextchar>
			</condition>
		</rule>

		<rule>
			<char><![CDATA[ʱ]]></char>
			<condition>
				<type>date</type>
				<pretype>digit, chardigit</pretype>
				<end>true</end>
			</condition>
		</rule>

		<rule>
			<char><![CDATA[.-]]></char>
			<condition>
				<type>letter</type>
				<pretype>digit,letter</pretype>
				<nexttype>letter</nexttype>
			</condition>
			<condition>
				<type>letter</type>
				<pretype>digit</pretype>
				<nexttype>letter</nexttype>
			</condition>
			<condition>
				<type>digit</type>
				<pretype>digit</pretype>
				<nexttype>digit</nexttype>
			</condition>
			<condition>
				<type>chardigit</type>
				<pretype>chardigit</pretype>
				<nexttype>chardigit</nexttype>
			</condition>
		</rule>
		
		<rule>
			<char><![CDATA[,]]></char>
			<condition>
				<type>digit</type>
				<pretype>digit</pretype>
				<nexttype>digit</nexttype>
			</condition>
		</rule>
		
		<rule>
			<char><![CDATA[֮ɰ]]></char>
			<condition>
				<type>chardigit</type>
				<pretype>chardigit</pretype>
				<nexttype>chardigit</nexttype>
			</condition>
		</rule>

		<rule>
			<char><![CDATA[]]></char>
			<condition>
				<type>chardigit</type>
				<pretype>chardigit</pretype>
				<end>true</end>
			</condition>
		</rule>
		
		<rule>
			<char><![CDATA[]]></char>
			<condition>
				<type>chardigit</type>
				<nopretype>chardigit</nopretype>
				<nexttype>chardigit</nexttype>
			</condition>
			<condition>
				<type>digit</type>
				<nopretype>digit</nopretype>
				<nexttype>digit</nexttype>
			</condition>
		</rule>
		
		<rule>
			<char><![CDATA[]]></char>
			<condition>
				<type>chardigit</type>
				<nopretype>chardigit</nopretype>
				<nexttype>chardigit</nexttype>
			</condition>
			<condition>
				<type>chardigit</type>
				<pretype>chardigit</pretype>
				<nexttype>chardigit</nexttype>
			</condition>
		</rule>

		<rule>
			<char><![CDATA[%]]></char>
			<condition>
				<type>chardigit</type>
				<pretype>chardigit</pretype>
				<end>true</end>
			</condition>
		</rule>

		<rule>
			<char><![CDATA[.-]]></char>
			<condition>
				<type>letter</type>
				<pretype>digit,letter</pretype>
				<nexttype>letter</nexttype>
			</condition>
			<condition>
				<type>letter</type>
				<pretype>digit</pretype>
				<nexttype>letter</nexttype>
			</condition>
			<condition>
				<type>digit</type>
				<pretype>digit</pretype>
				<nexttype>digit</nexttype>
			</condition>
			<condition>
				<type>chardigit</type>
				<pretype>chardigit</pretype>
				<nexttype>chardigit</nexttype>
			</condition>
		</rule>
		
		<rule>
			<char><![CDATA[,]]></char>
			<condition>
				<type>digit</type>
				<pretype>digit</pretype>
				<nexttype>digit</nexttype>
			</condition>
		</rule>
		
		<rule>
			<char><![CDATA[֮ɰ]]></char>
			<condition>
				<type>chardigit</type>
				<pretype>chardigit</pretype>
				<nexttype>chardigit</nexttype>
			</condition>
		</rule>

		<rule>
			<char><![CDATA[]]></char>
			<condition>
				<type>chardigit</type>
				<pretype>chardigit</pretype>
				<end>true</end>
			</condition>
		</rule>
		
		<rule>
			<char><![CDATA[]]></char>
			<condition>
				<type>chardigit</type>
				<nopretype>chardigit</nopretype>
				<nexttype>chardigit</nexttype>
			</condition>
			<condition>
				<type>digit</type>
				<nopretype>digit</nopretype>
				<nexttype>digit</nexttype>
			</condition>
		</rule>
		
		<rule>
			<char><![CDATA[]]></char>
			<condition>
				<type>chardigit</type>
				<nopretype>chardigit</nopretype>
				<nexttype>chardigit</nexttype>
			</condition>
		</rule>

		<rule>
			<char><![CDATA[%]]></char>
			<condition>
				<type>chardigit</type>
				<pretype>chardigit</pretype>
				<end>true</end>
			</condition>
			<condition>
				<type>digit</type>
				<pretype>digit</pretype>
				<end>true</end>
			</condition>
		</rule>
		
		<rule>
			<char><![CDATA[]]></char>
			<condition>
				<type>chardigit</type>
				<nopretype>chardigit</nopretype>
				<nextchar><![CDATA[ʮǧ]]></nextchar>
			</condition>
		</rule>
		
		<rule>
			<char><![CDATA[ʮǧ]]></char>
			<condition>
				<type>digit</type>
				<pretype>digit</pretype>
			</condition>
		</rule>

	</rules>

</cma>

{'fraction_non_alphanumeric': 0.22145522388059702, 'fraction_numerical': 0.003917910447761194, 'mean_word_length': 3.43424317617866, 'pattern_counts': {'":': 0, '<': 387, '<?xml version=': 1, '>': 387, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7518363', 'n_tokens_mistral': 2824, 'n_tokens_neox': 2168, 'n_words': 246}