<commit_before>{
  "capitalization": "case insensitive",
  "phraselist": [
    {
      "name": "Firearms",
      "options": {
        "editDistance": 1,
        "fuzzyMatchingMinimumLength": 4
      },
      "phrases": [
        {
          "phrase": "AR-15",
          "variations": [
            "AR15",
            "AR - 15"
          ]
          
        },
        {
          "phrase": "AK-47"
        }
      ]
    },
    {
      "name": "Firearms-technology",
      "options": {
        "editDistance": 1,
        "fuzzyMatchingMinimumLength": 4
      },
      "phrases": [
        {
          "phrase": "Glock Switch"
          
        },
        {
          "phrase": "LDC"
          
        }
      ]
    },
    {
      "name": "gang",
      "options": {
        "editDistance": 1,
        "fuzzyMatchingMinimumLength": 4
      },
      "phrases": [
        {
          "phrase": "clap-clap",
          "variations": [
            "clapclap",
            "clap clap"
          ]
        },
        {
          "phrase": "mr. 9mm",
          "variations": [
            "mr 9mm",
            "mr9mm"
          ]
        }
      ]
    },
    {
      "name": "non-english",
      "options": {
        "editDistance": 1,
        "fuzzyMatchingMinimumLength": 4
      },
      "phrases": [
        {
          "phrase": "Tolba"
          
        }
      ]
    }
  ]
}
<commit_msg>changed the wordlist format<commit_after>{
  "capitalization": "case insensitive",
  "phraselist": [
    {
      "name": "Firearms",
      "options": {
        "editDistance": 1,
        "fuzzyMatchingMinimumLength": 4
      },
      "phrases": [
        {
          "phrase": "AR-15",
          "variations": [
            "AR15",
            "AR - 15"
          ]
        },
        {
          "phrase": "5.56"
        },
        {
          "phrase": "7.62"
        },
        {
          "phrase": ".45 caliber"
        },
        {
          "phrase": ".50 caliber"
        },
        {
          "phrase": "37 mm",
          "variations": [
            "37mm"
          ]
        },
        {
          "phrase": "3-D firearm"
        },
        {
          "phrase": "3-D print"
        },
        {
          "phrase": "3-D printed"
        },
        {
          "phrase": "80 percent receiver",
          "variations": [
            "80% receiver",
            "80%receiver"
          ]
        },
        {
          "phrase": "9mm"
        },
        {
          "phrase": "ammo"
        },
        {
          "phrase": "ammunition"
        },
        {
          "phrase": "APA"
        },
        {
          "phrase": "AR",
          "variations": [
            "AR-15",
            "AR15"
          ]
        },
        {
          "phrase": "Armor piercing"
        },
        {
          "phrase": "Armor piercing ammuntion"
        },
        {
          "phrase": "assault rifle"
        },
        {
          "phrase": "ATF"
        },
        {
          "phrase": "ATFE"
        },
        {
          "phrase": "Barrett"
        },
        {
          "phrase": "BATFE"
        },
        {
          "phrase": "bullet proof vest"
        },
        {
          "phrase": "bump fire"
        },
        {
          "phrase": "bump stock"
        },
        {
          "phrase": "caliber"
        },
        {
          "phrase": "carbine"
        },
        {
          "phrase": "centerfire"
        },
        {
          "phrase": "Chopper"
        },
        {
          "phrase": "clip"
        },
        {
          "phrase": "CNC"
        },
        {
          "phrase": "concealable"
        },
        {
          "phrase": "firearm"
        },
        {
          "phrase": "derringer"
        },
        {
          "phrase": "fixed stock"
        },
        {
          "phrase": "flare gun"
        },
        {
          "phrase": "FMJ"
        },
        {
          "phrase": "folding stock"
        },
        {
          "phrase": "frame"
        },
        {
          "phrase": "full metal jacket"
        },
        {
          "phrase": "gas mask"
        },
        {
          "phrase": "ghost gun"
        },
        {
          "phrase": "Glock"
        },
        {
          "phrase": "handgun"
        },
        {
          "phrase": "hi point"
        },
        {
          "phrase": "high capacity"
        },
        {
          "phrase": "jimenez"
        },
        {
          "phrase": "keltec"
        },
        {
          "phrase": "kevlar"
        },
        {
          "phrase": "long gun"
        },
        {
          "phrase": "mag"
        },
        {
          "phrase": "magazine"
        },
        {
          "phrase": "MP4",
          "variations": [
            "MP5"
          ]
        },
        {
          "phrase": "oblit"
        },
        {
          "phrase": "obliterated"
        },
        {
          "phrase": "pistol"
        },
        {
          "phrase": "receiver"
        },
        {
          "phrase": "revolver"
        },
        {
          "phrase": "rifle"
        },
        {
          "phrase": "rimfire"
        },
        {
          "phrase": "RPG"
        },
        {
          "phrase": "S&W"
        },
        {
          "phrase": "SA"
        },
        {
          "phrase": "semi-auto"
        },
        {
          "phrase": "semi-automatic"
        },
        {
          "phrase": "shotgun"
        },
        {
          "phrase": "shotty"
        },
        {
          "phrase": "shoty"
        },
        {
          "phrase": "sig brace"
        },
        {
          "phrase": "stun gun"
        },
        {
          "phrase": "Taurus"
        },
        {
          "phrase": "uzi"
        },
        {
          "phrase": "vest"
        },
        {
          "phrase": "Arisaka"
        },
        {
          "phrase": "Mauser"
        },
        {
          "phrase": "MK43"
        },
        {
          "phrase": "K98"
        },
        {
          "phrase": "8mm"
        }
      ]
    },
    {
      "name": "Firearms-technology",
      "options": {
        "editDistance": 1,
        "fuzzyMatchingMinimumLength": 4
      },
      "phrases": [
        {
          "phrase": "Glock Switch"
        },
        {
          "phrase": "LDC"
        },
        {
          "phrase": "Chip"
        },
        {
          "phrase": "Baffle"
        },
        {
          "phrase": "Baffle Stack"
        },
        {
          "phrase": "Monocore"
        },
        {
          "phrase": "Anarchist"
        },
        {
          "phrase": "Solvent Trap"
        },
        {
          "phrase": "Belt Fed"
        },
        {
          "phrase": "Crew served"
        },
        {
          "phrase": "Hip Whip"
        },
        {
          "phrase": "Polymer"
        },
        {
          "phrase": "CNC machine"
        },
        {
          "phrase": "Green tip"
        },
        {
          "phrase": "Black tip"
        },
        {
          "phrase": "FAL"
        },
        {
          "phrase": "Outer tube"
        },
        {
          "phrase": "trigger"
        }
      ]
    },
    {
      "name": "gang",
      "options": {
        "editDistance": 1,
        "fuzzyMatchingMinimumLength": 4
      },
      "phrases": [
        {
          "phrase": "clap-clap",
          "variations": [
            "clapclap",
            "clap clap"
          ]
        },
        {
          "phrase": "mr. 9mm",
          "variations": [
            "mr 9mm",
            "mr9mm"
          ]
        },
        {
          "phrase": "boom stick"
        },
        {
          "phrase": "burner"
        },
        {
          "phrase": "cash, money, heroin",
          "variations": [
            "cash,money,heroin",
            "cash, money,heroin",
            "cash,money, heroin"
          ]
        },
        {
          "phrase": "chopper"
        },
        {
          "phrase": "dat fire"
        },
        {
          "phrase": "dat thang"
        },
        {
          "phrase": "davy crocket"
        },
        {
          "phrase": "deuce",
          "variations": [
            "duece duece",
            "duece"
          ]
        },
        {
          "phrase": "fofo"
        },
        {
          "phrase": "four nickle"
        },
        {
          "phrase": "gat",
          "variations": [
            "gatt"
          ]
        },
        {
          "phrase": "guage"
        },
        {
          "phrase": "ghost load"
        },
        {
          "phrase": "hammer"
        },
        {
          "phrase": "heater"
        },
        {
          "phrase": "hog leg"
        },
        {
          "phrase": "lead spitta"
        },
        {
          "phrase": "lil buddy"
        },
        {
          "phrase": "long pumps"
        },
        {
          "phrase": "mac"
        },
        {
          "phrase": "mack"
        },
        {
          "phrase": "narco"
        },
        {
          "phrase": "nina"
        },
        {
          "phrase": "nine"
        },
        {
          "phrase": "ol'betsy"
        },
        {
          "phrase": "ooh wop"
        },
        {
          "phrase": "piece"
        },
        {
          "phrase": "pocket rocket"
        },
        {
          "phrase": "rod"
        },
        {
          "phrase": "roscoe"
        },
        {
          "phrase": "sawdy"
        },
        {
          "phrase": "shotty"
        },
        {
          "phrase": "smoke wagon"
        },
        {
          "phrase": "strap"
        },
        {
          "phrase": "throw down"
        },
        {
          "phrase": "thunder stick"
        },
        {
          "phrase": "toaster"
        },
        {
          "phrase": "tres"
        },
        {
          "phrase": "widow maker"
        },
        {
          "phrase": "yeezy"
        }
      ]
    },
    {
      "name": "nfa",
      "options": {
        "editDistance": 1,
        "fuzzyMatchingMinimumLength": 4
      },
      "phrases": [
        {
          "phrase": "11.5 inch",
          "variations": [
            "11.5"
          ]
        },
        {
          "phrase": "14 inch",
          "variations": [
            "14"
          ]
        },
        {
          "phrase": "any other weapon"
        },
        {
          "phrase": "AOW"
        },
        {
          "phrase": "ATFE"
        },
        {
          "phrase": "auto sear",
          "variations": [
            "autosear"
          ]
        },
        {
          "phrase": "automatic"
        },
        {
          "phrase": "AW-SIM"
        },
        {
          "phrase": "BATFE"
        },
        {
          "phrase": "Bureau of ATF"
        },
        {
          "phrase": "cane gun"
        },
        {
          "phrase": "class 3",
          "variations": [
            "class III"
          ]
        },
        {
          "phrase": "conversion",
          "variations": [
            "conversion kit"
          ]
        },
        {
          "phrase": "destructive device"
        },
        {
          "phrase": "DIAS"
        },
        {
          "phrase": "drop in auto sear",
          "variations": [
            "drop in autosear"
          ]
        },
        {
          "phrase": "FA"
        },
        {
          "phrase": "flashbang",
          "variations": [
            "flash bang"
          ]
        },
        {
          "phrase": "forward grip"
        },
        {
          "phrase": "full auto"
        },
        {
          "phrase": "grenade"
        },
        {
          "phrase": "homemade silencer"
        },
        {
          "phrase": "homemade suppressor"
        },
        {
          "phrase": "incendiary"
        },
        {
          "phrase": "land mine"
        },
        {
          "phrase": "Lightning Link"
        },
        {
          "phrase": "machine gun"
        },
        {
          "phrase": "machinegun"
        },
        {
          "phrase": "missile"
        },
        {
          "phrase": "molotov",
          "variations": [
            "molotov cocktail"
          ]
        },
        {
          "phrase": "muffler"
        },
        {
          "phrase": "nfa"
        },
        {
          "phrase": "pen gun"
        },
        {
          "phrase": "pistol grip"
        },
        {
          "phrase": "poison gas"
        },
        {
          "phrase": "RLL"
        },
        {
          "phrase": "rocket",
          "variations": [
            "rocket launcher"
          ]
        },
        {
          "phrase": "sawed off"
        },
        {
          "phrase": "sbr"
        },
        {
          "phrase": "sbs"
        },
        {
          "phrase": "sear"
        },
        {
          "phrase": "short barrel rifle",
          "variations": [
            "short barrel shotgun",
            "short barreled rifle",
            "short barreled shotgun"
          ]
        },
        {
          "phrase": "silencer"
        },
        {
          "phrase": "smooth bore"
        },
        {
          "phrase": "SOT"
        },
        {
          "phrase": "Special Occupational Tax"
        },
        {
          "phrase": "street sweeper"
        },
        {
          "phrase": "suppressor"
        },
        {
          "phrase": "umbrella gun"
        },
        {
          "phrase": "weapon made from"
        },
        {
          "phrase": "Zip gun"
        },
        {
          "phrase": "Zipgun"
        },
        {
          "phrase": "military"
        }
      ]
    },
    {
      "name": "non-english",
      "options": {
        "editDistance": 1,
        "fuzzyMatchingMinimumLength": 4
      },
      "phrases": [
        {
          "phrase": "arma larga"
        },
        {
          "phrase": "armas de fuego"
        },
        {
          "phrase": "bala"
        },
        {
          "phrase": "cahuetas"
        },
        {
          "phrase": "Cartucho"
        },
        {
          "phrase": "Cebolla / Cebollas"
        },
        {
          "phrase": "cortas"
        },
        {
          "phrase": "cuerno de chivo",
          "variations": [
            "cuerno de chivo",
            "Cuerno de Chivo",
            "Cuerno"
          ]
        },
        {
          "phrase": "cuete"
        },
        {
          "phrase": "El Subfusil ametralladora"
        },
        {
          "phrase": "escopete"
        },
        {
          "phrase": "escuadra"
        },
        {
          "phrase": "explosivo"
        },
        {
          "phrase": "fusca"
        },
        {
          "phrase": "gat"
        },
        {
          "phrase": "la ametralladora"
        },
        {
          "phrase": "la escopeta"
        },
        {
          "phrase": "Lanza papas"
        },
        {
          "phrase": "largas"
        },
        {
          "phrase": "municion",
          "variations": [
            "Municiones"
          ]
        },
        {
          "phrase": "Papa / Papas"
        },
        {
          "phrase": "penas"
        },
        {
          "phrase": "Pertrechos"
        },
        {
          "phrase": "pistola"
        },
        {
          "phrase": "Polvora"
        },
        {
          "phrase": "proyectil"
        },
        {
          "phrase": "r quinze"
        },
        {
          "phrase": "trick or treat"
        },
        {
          "phrase": "tricki-tricki"
        },
        {
          "phrase": "Vaina"
        }
      ]
    }
  ]
}

{'fraction_non_alphanumeric': 0.15235528173533563, 'fraction_numerical': 0.005390835579514825, 'mean_word_length': 0.7726083494482994, 'pattern_counts': {'":': 302, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6433860', 'n_tokens_mistral': 4826, 'n_tokens_neox': 4695, 'n_words': 741}