<commit_before>#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Contraction filtering
#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Inserting a space before the English possessive and other contractions (and other accidental matches)
# so they (including the apostrophe) becomes their own tokens.
# so text like
#
#    we're heading to the park ...    => [ we, 're, heading, to, the, park ]
#    Mexico's valiant effort to...    => [ mexico, 's, valiant, effort, to ]
# etc.
#
"'s" => " 's"
"'ve" => " 've"
"'re" => " 're"
"'ll" => " 'll"






#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Latin-1 to ASCII mappings;  as well as Unicode-to-ASCII mapping
# Aka character folding.
# SOURCE:  solr 4.2.1 mapping-ASCIIFolding*txt
# EDITS:  see "OMIT" ommissions below for select A and O chars.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0<commit_msg>advanced: Apos+s is important to remap to 's, as Unicode Apostrophe is found often enough at the end of place names<commit_after>#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Contraction filtering
#
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Inserting a space before the English possessive and other contractions (and other accidental matches)
# so they (including the apostrophe) becomes their own tokens.
# so text like
#
#    we're heading to the park ...    => [ we, 're, heading, to, the, park ]
#    Mexico's valiant effort to...    => [ mexico, 's, valiant, effort, to ]
# etc.
#
"\u2019s" => " 's"
"'s" => " 's"
"'ve" => " 've"
"'re" => " 're"
"'ll" => " 'll"






#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Latin-1 to ASCII mappings;  as well as Unicode-to-ASCII mapping
# Aka character folding.
# SOURCE:  solr 4.2.1 mapping-ASCIIFolding*txt
# EDITS:  see "OMIT" ommissions below for select A and O chars.
#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
{'fraction_non_alphanumeric': 0.2755144897102058, 'fraction_numerical': 0.008399832003359933, 'mean_word_length': 4.316964285714286, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3953398', 'n_tokens_mistral': 753, 'n_tokens_neox': 675, 'n_words': 265}