<commit_before># This is an example input file for mkbkindex.
# These lines (with preceding '#') are treated as comments.

# Following any lines must be encoded by Shift_JIS.
# t@C͍̓s̓Vtg JIS ŕĂKv܂B
# UTF-8 Ȃǂ̏ꍇ́A炩߃Vtg JIS ɕϊĂĂB

# We're afraid that the following explanation is written in Japanese
# because this tool is optimized for Japanese book publishing.

# tH[}bg:
# 
# ipXy[XĂǂj 1ȏ̃^uiXy[Xsj mu̗iXy[XĂ͂Ȃj
# 
# A
# 
# ipXy[XĂǂj[}] 1ȏ̃^uiXy[Xsj mu̗iXy[XĂ͂Ȃj
# 
# łB}ꒆ ...  ` Ɏu܂B
# 
# E 擪 # Ŏn܂s̓RgłB#@ ͗\񂳂Ă̂
#    gȂłB
# 
# E śAŏIIɂ͎菜܂B
#
# GNU Emacs gꍇATAB L[ tab-to-tab-stop 蓖ĂĂ
# ֗łB
#
# : M-x local-set-key TAB tab-to-tab-stop
#
# 
# ȉ̓Tv͂łB
#

vO~O			65,x,iv
3xZT[		81
fobO			23,73
				38
operating system		ii,43
fobO			iii
f[^[...]		46,77
f[^[...o]		56,83
񕪖				85,37
f[^[@B̏o͂...]	96
f[^				66,28

# s@:
#
# mkbkindex input.txt
#

# ʂ́Afinal.out Ƃďo͂܂Błɓ̃t@Cꍇ
# ɂ́A㏑̑OɊmF߂܂B<commit_msg>Fixed a comment line<commit_after># This is an example input file for mkbkindex.
# These lines (with preceding '#') are treated as comments.

# Following any lines must be encoded by Shift_JIS.
# t@C͍̓s̓Vtg JIS ŕĂKv܂B
# UTF-8 Ȃǂ̏ꍇ́A炩߃Vtg JIS ɕϊĂĂB

# We're afraid that the following explanation is written in Japanese
# because this tool is optimized for Japanese book publishing.

# tH[}bg:
# 
# ipXy[XĂǂj 1ȏ̃^uiXy[Xsj mu̗iXy[XĂ͂Ȃj
# 
# A
# 
# ipXy[XĂǂj[}] 1ȏ̃^uiXy[Xsj mu̗iXy[XĂ͂Ȃj
# 
# łB}ꒆ ...  ` Ɏu܂B
# 
# E 擪 # Ŏn܂s̓RgłB#@ ͗\񂳂Ă̂
#    gȂłB
# 
# E śAŏIIɂ͎菜܂B
#
# GNU Emacs gꍇATAB L[ tab-to-tab-stop 蓖ĂĂ
# ֗łB
#
# : M-x local-set-key TAB tab-to-tab-stop
#

# ȉ̓Tv͂łB
#

vO~O			65,x,iv
3xZT[		81
fobO			23,73
				38
operating system		ii,43
fobO			iii
f[^[...]		46,77
f[^[...o]		56,83
񕪖				85,37
f[^[@B̏o͂...]	96
f[^				66,28

# s@:
#
# mkbkindex input.txt
#

# ʂ́Afinal.out Ƃďo͂܂Błɓ̃t@Cꍇ
# ɂ́A㏑̑OɊmF߂܂B
{'fraction_non_alphanumeric': 0.17926089354660782, 'fraction_numerical': 0.039713182570325425, 'mean_word_length': 3.2284382284382285, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2531913', 'n_tokens_mistral': 1226, 'n_tokens_neox': 1177, 'n_words': 218}