<commit_before>#friso lexicon configure file.
# @email	<EMAIL>
# @date		2012-12-19
#main lexion
__LEX_CJK_WORDS__ :[
	lex-main.lex;
	lex-admin.lex;
	lex-chars.lex;
	lex-cn-mz.lex;
	lex-cn-place.lex;
	lex-company.lex;
	lex-festival.lex;
	lex-flname.lex;
	lex-food.lex;
	lex-lang.lex;
	lex-nation.lex;
	lex-net.lex;
	lex-org.lex;
#add more here
]
#single chinese unit lexicon
__LEX_CJK_UNITS__ :[
	lex-units.lex;
]
#chinese and english mixed word lexicon like "b超".
__LEX_ECM_WORDS__:[
	lex-ecmix.lex;
]
#english and chinese mixed word lexicon like "卡拉ok".
__LEX_CEM_WORDS__:[
	lex-cemix.lex;
]
#chinese last name lexicon.
__LEX_CN_LNAME__:[
	lex-lname.lex;
]
#single name words lexicon.<commit_msg>最新版本的friso.lex.ini<commit_after>#friso lexicon configure file.
# @email	<EMAIL>
# @date		2012-12-19
#main lexion
__LEX_CJK_WORDS__ :[
	lex-main.lex;
	lex-admin.lex;
	lex-chars.lex;
	lex-cn-mz.lex;
	lex-cn-place.lex;
	lex-company.lex;
	lex-festival.lex;
	lex-flname.lex;
	lex-food.lex;
	lex-lang.lex;
	lex-nation.lex;
	lex-net.lex;
	lex-org.lex;
	lex-touris.lex;
#add more here
]
#single chinese unit lexicon
__LEX_CJK_UNITS__ :[
	lex-units.lex;
]
#chinese and english mixed word lexicon like "b超".
__LEX_ECM_WORDS__:[
	lex-ecmix.lex;
]
#english and chinese mixed word lexicon like "卡拉ok".
__LEX_CEM_WORDS__:[
	lex-cemix.lex;
]
#chinese last name lexicon.
__LEX_CN_LNAME__:[
	lex-lname.lex;
]
#single name words lexicon.
{'fraction_non_alphanumeric': 0.13912429378531074, 'fraction_numerical': 0.011299435028248588, 'mean_word_length': 4.228782287822878, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3468098', 'n_tokens_mistral': 698, 'n_tokens_neox': 657, 'n_words': 126}