<commit_before># English confusion sets, mostly homophones
# Original source: After the Deadline's data/rules/homophonedb.txt
# URL: http://openatd.wordpress.com/ or https://openatd.svn.wordpress.org/atd-server/data/rules/homophonedb.txt
# License: GNU GENERAL PUBLIC LICENSE Version 2
#
# Line format:
# <word1>|<description1>; <word2>|<description2>; <factor>   # optional comment
#   <word1> and <word2> are words that can easily be confused
#   <description> will be used in the error message to explain the word (optional)
#   <factor> is the factor of how much more the other word must be more
#            probable so the text is considered potentially incorrect.
#            Use a higher value for better precision but lower recall.
#   Precision and recall values in the comments come from ConfusionRuleEvaluator
#     and are based on Wikipedia and Tatoeba data, usually with 1000 random sentences
#     (if that many sentence are available for that pair). Starting from 2015-05-30,
#     the precision/recall values are from a Wikipedia/Tatoeba/Enron mix.

and; end; 100000000000                                                           # precision=0.998, recall=0.484 (2015-07-29) using 3grams
accept|to agree, to endure; except|with the exception of; 10000                  # precision=0.998, recall=0.787 (2015-05-25)
ate|past of 'to eat'; eight|the number '8'; 10                                   # precision=0.998, recall=0.882 (2015-05-25)
bean; been; 100                                                                  # precision=0.998, recall=0.949 (2015-05-27)
breathe|verb; breath|noun; 10000                                                 # precision=0.993, recall=0.796 (2015-05-27) actually 0 false alarms
but; bit; 1000000                                                                # precision=0.997, recall=0.753 (2015-06-13) using 3grams
but; butt; 10000                                                                 # precision=0.999, recall=0.890 (2015-05-27) false alarm: butt in
buy|purchase; bye|goodbye; 100                                                   # precision=0.999, recall=0.975 (2015-05-27)
dessert|last course of a meal; desert|arid land; 100000                          # precision=0.996, recall=0.558 (2015-05-27)
effect|outcome, result; affect|experience of emotion; 1000000                    # precision=0.996, recall=0.685 (2015-05-27)
extent|noun; extend|verb; 10                                                     # precision=0.998, recall=0.976 (2015-05-25)
first; fist; 100000000                                                           # precision=1.000, recall=0.816 (2015-05-27)
four|the number '4';for|a preposition as in 'I have something for you.'; 10000   # precision=0.996, recall=0.486 (2015-05-22)
full; fill; 100000                                                               # precision=0.996, recall=0.792 (2015-05-30) using 3grams. false alarm: fill-in
know|to be aware of; now|in this moment; 100                                     # precision=0.997, recall=0.877 (2015-05-22)
had|past of 'have'; hat|head covering; 10000000                                  # precision=0.998, recall=0.646 (2015-07-29) using 3grams
lines; lies; 100000                                                              # precision=0.999, recall=0.558 (2015-07-29) using 3grams
loose|not tight; lose|opposite of win; 10000                                     # precision=0.996, recall=0.816 (2015-05-29)
neat|clean or organized; near|close to; 10000                                    # precision=1.000, recall=0.783 (2015-07-29) using 3grams
news; new; 100000                                                                # precision=0.998, recall=0.578 (2015-05-29)
nice|pleasant; mice|plural of 'mouse'; 100                                       # precision=1.000, recall=0.932 (2015-05-25)
nit; not; 10000000                                                               # precision=0.998, recall=0.945 (2015-07-29) using 3grams
no|negation; now|at the present moment; 100000                                   # precision=0.996, recall=0.564 (2015-07-29) using 3grams
note; not; 10000000                                                              # precision=0.995, recall=0.537 (2015-06-13) using 3grams
our; out; 1000                                                                   # precision=0.999, recall=0.910 (2015-05-27)
peek|a secret look; peak|extremum, summit, top; 10000                            # precision=1.000, recall=0.799 (2015-06-13) using 3grams
pray|say a prayer; prey|animal hunted for food; 1000                             # precision=0.996, recall=0.763 (2015-05-25)
proof|often noun, also 'proofread';prove|verb: show, testify, demonstrate; 10000 # precision=0.996, recall=0.856 (2015-05-29)
rite|established ceremony or practice; right|opposite of 'left'; 10000000        # precision=0.996, recall=0.751 (2015-05-26) false alarms: "...of the **rite."
same|identical; sane|mentally healthy; 1000                                      # precision=1.000, recall=0.971 (2015-07-29) using 3grams
see|to look at; sea|ocean; 1000                                                  # precision=0.998, recall=0.877 (2015-06-01) using 3grams
their|as in 'It’s not their fault.'; there|as in 'Is there an answer?'; 100      # precision=0.998, recall=0.970 (2015-05-25) false alarms: there being
than; that; 10000000                                                             # precision=0.999, recall=0.502 (2015-07-29) using 3grams
then|next in order; than|used in comparisons; 10000                              # precision=0.997, recall=0.766 (2015-05-27)
think|usually a verb; thing|noun; 100                                            # precision=0.998, recall=0.961 (2015-05-30) using 3grams
through|preposition, e.g. 'go through the door'; threw|past of throw; 10000000   # precision=0.998, recall=0.580 (2015-06-13) using 3grams
to; the; 1000000                                                                 # precision=0.999, recall=0.644 (2015-05-29)
whether|as in 'whether or not'; weather|atmospheric conditions; 10000            # precision=0.997, recall=0.914 (2015-05-29) false alarm: weather conditions
you|personal pronoun; your|possessive pronoun; 1000                              # precision=0.999, recall=0.947 (2015-05-25)

# results on a better evaluation corpus with wikipedia, tatoeba, enron (33% each):
# accept/except precision=0.997, recall=0.784 (2015-05-29)
# first/fist precision=0.992, recall=0.771 (2015-05-29)
# our/out precision=0.998, recall=0.863 (2015-05-29)
# their/there precision=0.992, recall=0.945 (2015-05-29)
# whether/weather precision=0.991, recall=0.806 (2015-05-29)

#in; on; 10000                                                                   # precision=0.975, recall=0.313 (2015-06-01) using 3grams
#its; it; 10000000                                                               # precision=0.991, recall=0.639 (2015-06-01) using 3grams - "it's" must also be supported to avoid wrong suggestions
#if; is; 1000000                                                                 # precision=0.993, recall=0.529 (2015-06-01) using 3grams<commit_msg>[en] adding more confusion pairs: know/known; no/know<commit_after># English confusion sets, mostly homophones
# Original source: After the Deadline's data/rules/homophonedb.txt
# URL: http://openatd.wordpress.com/ or https://openatd.svn.wordpress.org/atd-server/data/rules/homophonedb.txt
# License: GNU GENERAL PUBLIC LICENSE Version 2
#
# Line format:
# <word1>|<description1>; <word2>|<description2>; <factor>   # optional comment
#   <word1> and <word2> are words that can easily be confused
#   <description> will be used in the error message to explain the word (optional)
#   <factor> is the factor of how much more the other word must be more
#            probable so the text is considered potentially incorrect.
#            Use a higher value for better precision but lower recall.
#   Precision and recall values in the comments come from ConfusionRuleEvaluator
#     and are based on Wikipedia and Tatoeba data, usually with 1000 random sentences
#     (if that many sentence are available for that pair). Starting from 2015-05-30,
#     the precision/recall values are from a Wikipedia/Tatoeba/Enron mix.

and; end; 100000000000                                                           # precision=0.998, recall=0.484 (2015-07-29) using 3grams
accept|to agree, to endure; except|with the exception of; 10000                  # precision=0.998, recall=0.787 (2015-05-25)
ate|past of 'to eat'; eight|the number '8'; 10                                   # precision=0.998, recall=0.882 (2015-05-25)
bean; been; 100                                                                  # precision=0.998, recall=0.949 (2015-05-27)
breathe|verb; breath|noun; 10000                                                 # precision=0.993, recall=0.796 (2015-05-27) actually 0 false alarms
but; bit; 1000000                                                                # precision=0.997, recall=0.753 (2015-06-13) using 3grams
but; butt; 10000                                                                 # precision=0.999, recall=0.890 (2015-05-27) false alarm: butt in
buy|purchase; bye|goodbye; 100                                                   # precision=0.999, recall=0.975 (2015-05-27)
dessert|last course of a meal; desert|arid land; 100000                          # precision=0.996, recall=0.558 (2015-05-27)
effect|outcome, result; affect|experience of emotion; 1000000                    # precision=0.996, recall=0.685 (2015-05-27)
extent|noun; extend|verb; 10                                                     # precision=0.998, recall=0.976 (2015-05-25)
first; fist; 100000000                                                           # precision=1.000, recall=0.816 (2015-05-27)
four|the number '4';for|a preposition as in 'I have something for you.'; 10000   # precision=0.996, recall=0.486 (2015-05-22)
full; fill; 100000                                                               # precision=0.996, recall=0.792 (2015-05-30) using 3grams. false alarm: fill-in
know|to be aware of; now|in this moment; 100                                     # precision=0.997, recall=0.877 (2015-05-22)
know|to be aware of; known|past participle of 'know'; 100                        # precision=0.999, recall=0.965 (2015-07-29) using 3grams
had|past of 'have'; hat|head covering; 10000000                                  # precision=0.998, recall=0.646 (2015-07-29) using 3grams
lines; lies; 100000                                                              # precision=0.999, recall=0.558 (2015-07-29) using 3grams
loose|not tight; lose|opposite of win; 10000                                     # precision=0.996, recall=0.816 (2015-05-29)
neat|clean or organized; near|close to; 10000                                    # precision=1.000, recall=0.783 (2015-07-29) using 3grams
news; new; 100000                                                                # precision=0.998, recall=0.578 (2015-05-29)
nice|pleasant; mice|plural of 'mouse'; 100                                       # precision=1.000, recall=0.932 (2015-05-25)
nit; not; 10000000                                                               # precision=0.998, recall=0.945 (2015-07-29) using 3grams
no|negation; now|at the present moment; 100000                                   # precision=0.996, recall=0.564 (2015-07-29) using 3grams
no|negation; know|to be aware of; 10000                                          # precision=0.999, recall=0.888 (2015-07-29) using 3grams
note; not; 10000000                                                              # precision=0.995, recall=0.537 (2015-06-13) using 3grams
our; out; 1000                                                                   # precision=0.999, recall=0.910 (2015-05-27)
peek|a secret look; peak|extremum, summit, top; 10000                            # precision=1.000, recall=0.799 (2015-06-13) using 3grams
pray|say a prayer; prey|animal hunted for food; 1000                             # precision=0.996, recall=0.763 (2015-05-25)
proof|often noun, also 'proofread';prove|verb: show, testify, demonstrate; 10000 # precision=0.996, recall=0.856 (2015-05-29)
rite|established ceremony or practice; right|opposite of 'left'; 10000000        # precision=0.996, recall=0.751 (2015-05-26) false alarms: "...of the **rite."
same|identical; sane|mentally healthy; 1000                                      # precision=1.000, recall=0.971 (2015-07-29) using 3grams
see|to look at; sea|ocean; 1000                                                  # precision=0.998, recall=0.877 (2015-06-01) using 3grams
their|as in 'It’s not their fault.'; there|as in 'Is there an answer?'; 100      # precision=0.998, recall=0.970 (2015-05-25) false alarms: there being
than; that; 10000000                                                             # precision=0.999, recall=0.502 (2015-07-29) using 3grams
then|next in order; than|used in comparisons; 10000                              # precision=0.997, recall=0.766 (2015-05-27)
think|usually a verb; thing|noun; 100                                            # precision=0.998, recall=0.961 (2015-05-30) using 3grams
through|preposition, e.g. 'go through the door'; threw|past of throw; 10000000   # precision=0.998, recall=0.580 (2015-06-13) using 3grams
to; the; 1000000                                                                 # precision=0.999, recall=0.644 (2015-05-29)
whether|as in 'whether or not'; weather|atmospheric conditions; 10000            # precision=0.997, recall=0.914 (2015-05-29) false alarm: weather conditions
you|personal pronoun; your|possessive pronoun; 1000                              # precision=0.999, recall=0.947 (2015-05-25)

# results on a better evaluation corpus with wikipedia, tatoeba, enron (33% each):
# accept/except precision=0.997, recall=0.784 (2015-05-29)
# first/fist precision=0.992, recall=0.771 (2015-05-29)
# our/out precision=0.998, recall=0.863 (2015-05-29)
# their/there precision=0.992, recall=0.945 (2015-05-29)
# whether/weather precision=0.991, recall=0.806 (2015-05-29)

#in; on; 10000                                                                   # precision=0.975, recall=0.313 (2015-06-01) using 3grams
#its; it; 10000000                                                               # precision=0.991, recall=0.639 (2015-06-01) using 3grams - "it's" must also be supported to avoid wrong suggestions
#if; is; 1000000                                                                 # precision=0.993, recall=0.529 (2015-06-01) using 3grams
{'fraction_non_alphanumeric': 0.10620070709817786, 'fraction_numerical': 0.1423714985042154, 'mean_word_length': 1.7218726868985936, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 18, 'word_list_counts': {'cursed_substrings.json': 116, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3408579', 'n_tokens_mistral': 5410, 'n_tokens_neox': 3911, 'n_words': 1190}