<commit_before># This Python file uses the following encoding: utf-8

charsAfterOpenCcConversion_To_CharsShownInSimplifiedText = {
# Conversion for the quotation marks.
		u"「":u"“",
		u"」":u"”",
		u"『":u"‘",
		u"』":u"’",
# OpenCC successfully converts the following characters to the simplified UTF-8 code.
# However, these simplified characters could not be shown in most of Chinese Font families.
# Thus, use the original traditional characters.
## Mengzi 
		u"𫗦":u"餔",
		u"𫛞":u"鴃",
		u"𫍙":u"訑",
## Zhongyong
		u"𫓧":u"鈇",
## Lunyu
		u"𫐐":u"輗",
		u"𫐄":u"軏",
		u"𫄨":u"絺",
# The following words converted by OpenCC could not reflect its original syntax meaning.
# Thus, modify words properly.
##　Mengzi
		u"将彻":u"将徹",
		u"以旗":u"以旂",
		u"关弓":u"弯弓",
		u"征招":u"徵招",
		u"遏籴":u"遏糴",
		u"逾":u"踰",
##　Lunyu
		u"后雕":u"后彫",
		u"不足征":u"不足徵",
		u"征之矣":u"徵之矣",
		u"絏":u"绁",
		u"不至于谷":u"不至于穀",
		u"道谷":u"道穀",
		u"旧谷":u"旧穀",
		u"新谷":u"新穀",
		u"五谷不分":u"五穀不分",
## Daxue
		u"恂栗":u"恂慄",
## Zhongyong
		u"半涂":u"半途",
		u"复帱":u"覆帱",
}

def Convert(content):
	for key in charsAfterOpenCcConversion_To_CharsShownInSimplifiedText:
		content = content.replace(key, charsAfterOpenCcConversion_To_CharsShownInSimplifiedText[key])
	return remove_redundant_subscript(content)

## 一對通假字在正體字中可以不同，但在簡體字中是一個字。所以在簡體版中，通假字標註是原字。應該去除冗餘的通假標註。
## 比如：舍~捨~：舍在正體字中是住處，比如宿舍。在經文中同“捨”，捨棄意。但是简体字中， 只有一个“舍”字，表达两个意思。
## 这样在简体字通假就失去了意义。經過轉換變成了“舍~舍~”。
## 又如：與~歟~ 這對通假字在簡體字中是通一個字。
def remove_redundant_subscript(content):
	i = 0
	while i < len(content) - 3:
		if content[i+1] == '~' and content[i+3] == '~' and content[i] == content[i+2]:
			content = content[:i+1] + content[i+4:]
		i += 1
	return content

# Todo: Complete this list in a different file.
checkedToneChars = u"入日"

# 入聲字： 用`^^`標誌, 比如： ^曰^。This is interpreted by [pandoc subscription 
#　extension](https://pandoc.org/MANUAL.html#extension-superscript-subscript).
def wrap_checkedToneChar_with_superscript(content):
	result = ""
	superscriptCharInPandoc = '^'
	for c in content:
		result += superscriptCharInPandoc + c + superscriptCharInPandoc if c in checkedToneChars else c
	return result
<commit_msg>working<commit_after># This Python file uses the following encoding: utf-8

charsAfterOpenCcConversion_To_CharsShownInSimplifiedText = {
# Conversion for the quotation marks.
		u"「":u"“",
		u"」":u"”",
		u"『":u"‘",
		u"』":u"’",
# OpenCC successfully converts the following characters to the simplified UTF-8 code.
# However, these simplified characters could not be shown in most of Chinese Font families.
# Thus, use the original traditional characters.
## Mengzi 
		u"𫗦":u"餔",
		u"𫛞":u"鴃",
		u"𫍙":u"訑",
## Zhongyong
		u"𫓧":u"鈇",
## Lunyu
		u"𫐐":u"輗",
		u"𫐄":u"軏",
		u"𫄨":u"絺",
# The following words converted by OpenCC could not reflect its original syntax meaning.
# Thus, modify words properly.
##　Mengzi
		u"将彻":u"将徹",
		u"以旗":u"以旂",
		u"关弓":u"弯弓",
		u"征招":u"徵招",
		u"遏籴":u"遏糴",
		u"逾":u"踰",
##　Lunyu
		u"后雕":u"后彫",
		u"不足征":u"不足徵",
		u"征之矣":u"徵之矣",
		u"絏":u"绁",
		u"不至于谷":u"不至于穀",
		u"道谷":u"道穀",
		u"旧谷":u"旧穀",
		u"新谷":u"新穀",
		u"五谷不分":u"五穀不分",
## Daxue
		u"恂栗":u"恂慄",
## Zhongyong
		u"半涂":u"半途",
		u"复帱":u"覆帱",
}

def Convert(content):
	for key in charsAfterOpenCcConversion_To_CharsShownInSimplifiedText:
		content = content.replace(key, charsAfterOpenCcConversion_To_CharsShownInSimplifiedText[key])
	return remove_redundant_subscript(content)

## 一對通假字在正體字中可以不同，但在簡體字中是一個字。所以在簡體版中，通假字標註是原字。應該去除冗餘的通假標註。
## 比如：舍~捨~：舍在正體字中是住處，比如宿舍。在經文中同“捨”，捨棄意。但是简体字中， 只有一个“舍”字，表达两个意思。
## 这样在简体字通假就失去了意义。經過轉換變成了“舍~舍~”。
## 又如：與~歟~ 這對通假字在簡體字中是通一個字。
def remove_redundant_subscript(content):
	i = 0
	while i < len(content) - 3:
		if content[i+1] == '~' and content[i+3] == '~' and content[i] == content[i+2]:
			content = content[:i+1] + content[i+4:]
		i += 1
	return content

# Todo: Complete this list in a different file.
checkedToneChars = u"入日"

# 入聲字： 用`^^`標誌, 比如： ^曰^。This is interpreted by [pandoc subscription 
#　extension](https://pandoc.org/MANUAL.html#extension-superscript-subscript).
def wrap_checkedToneChar_with_superscript(content):
	result = ""
	superscriptSignInPandoc = '^'
	for c in content:
		result += superscriptSignInPandoc + c + superscriptSignInPandoc if c in checkedToneChars else c
	return result

{'fraction_non_alphanumeric': 0.1720226843100189, 'fraction_numerical': 0.004725897920604915, 'mean_word_length': 5.502304147465438, 'pattern_counts': {'":': 58, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5359422', 'n_tokens_mistral': 2419, 'n_tokens_neox': 2314, 'n_words': 356}