<commit_before>{
  "words": [
    "typical",
    "typically",
    "slight",
    "slightly",
    "particularly",
    "commonly",
    "really",
    "seriously",
    "feel",
    "just",
    "lots",
    "virtually",
    "so",
    "very",
    "quite",
    "good",
    "stuff",
    "nice",
    "thing",
    "always",
    "never",
    "got",
    "literally",
    "actually",
    "totally",
    "essentially",
    "basically",
    "interesting",
    "irregardless",
    "nature",
    "plus",
    "utilize",
    "utilizes",
    "utilized"
  ],
  "phrases": {
    "in recent years": "recently",
    "and so on": "etc.",
    "a lot": "too vague & colloquial, be specific",
    "lots of": "too vague & colloquial, be specific",
    "sort of": "too vague",
    "kind of": "too vague",
    "used to": "consider simple past tense",
    "of course" : "consider: clearly, definitely, indeed, naturally, surely.",
    "find out": "consider: examined, investigated, interrogated, discovered, realized",
    "serves to" : "typically unnecessary",
    "helps to" : "typically unnecessary",
    "and also": "often redundant",
    "as to whether": "whether",
    "being that" : "being",
    "there are": "boring, rephrase",
    "there is" : "boring, rephrase",
    "considered to be" : "considered",
    "due to the fact that": "because or due to",
    "each and every": "use either each or every",
    "equally as": "equally",
    "had ought" : "ought",
    "in terms of" : "typically redundant",
    "on account of": "because",
    "so as to": "to",
    "suppose to": "didn't you mean supposed to?",
    "the reason why is because": "because",
    "pros and cons": "advantages and disadvantages",
    "this day": "presently, today",
    "these days": "presently, today",
    "everyday life": "cut completely or be specific",
    "is known to": "is + proper citation",
    "are known to": "are + proper citation",
    "chose to adopt": "use"
  }
}<commit_msg>Updated list of filter phrases<commit_after>{
  "words": [
    "typical",
    "typically",
    "slight",
    "slightly",
    "particularly",
    "commonly",
    "really",
    "seriously",
    "feel",
    "just",
    "lots",
    "virtually",
    "so",
    "very",
    "quite",
    "good",
    "stuff",
    "nice",
    "thing",
    "always",
    "never",
    "got",
    "literally",
    "actually",
    "totally",
    "essentially",
    "basically",
    "interesting",
    "irregardless",
    "nature",
    "plus",
    "utilize",
    "utilizes",
    "utilized",
    "important",
    "methodologic"
  ],
  "phrases": {
    "in recent years": "recently",
    "and so on": "etc.",
    "a lot": "too vague & colloquial, be specific",
    "lots of": "too vague & colloquial, be specific",
    "sort of": "too vague",
    "kind of": "too vague",
    "used to": "consider simple past tense",
    "of course" : "consider: clearly, definitely, indeed, naturally, surely",
    "find out": "consider: examined, investigated, interrogated, discovered, realized",
    "serves to" : "typically unnecessary",
    "helps to" : "typically unnecessary",
    "and also": "often redundant",
    "as to whether": "whether",
    "being that" : "being",
    "there are": "boring, consider rephrasing",
    "there is" : "boring, consider rephrasing",
    "considered to be" : "considered",
    "due to the fact that": "because or due to",
    "each and every": "use either each or every",
    "equally as": "equally",
    "had ought" : "ought",
    "in terms of" : "typically redundant",
    "on account of": "because",
    "so as to": "to",
    "suppose to": "didn't you mean supposed to?",
    "the reason why is because": "because",
    "pros and cons": "advantages and disadvantages",
    "this day": "presently, today",
    "these days": "presently, today",
    "everyday life": "cut completely or be specific",
    "is known to": "consider: is (or other simple verb) + proper citation",
    "are known to": "consider: are (or other simple verb) + proper citation",
    "has been shown to" : "consider: is (or other simple verb) + proper citation",
    "have been shown to" : "consider: are (or other simple verb) + proper citation",
    "were shown to": "consider: are (or other simple verb) + proper citation",
    "was shown to" : "consider: is (or other simple verb) + proper citation",
    "chose to adopt": "use",
    "successful solution": "are there unsuccessful solutions?",
    "successful solutions": "are there unsuccessful solutions?",
    "well known" : "consider removing and adding proper citation",
    "first of its kind": "consider simply: first",
    "can be regarded as": "vague, consider rephrasing",
    "it should be emphasized that": "long & boring, consider rephrasing, e.g,: we emphasize",
    "a number of": "many",
    "are of the same opinion": "agree",
    "less frequently occurring": "rare",
    "all three of the": "the three",
    "all two of": "both",
    "give raise to": "cause",
    "based on the assumption that": "if"
  }
}
{'fraction_non_alphanumeric': 0.1636546184738956, 'fraction_numerical': 0.0, 'mean_word_length': 3.0495934959349595, 'pattern_counts': {'":': 67, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7449253', 'n_tokens_mistral': 1672, 'n_tokens_neox': 1567, 'n_words': 556}