<commit_before>{
  "tables": [
    "unicode.dis",
    "fi.ctb"
  ],

  "tests": [
    {
      "data": [
        {
          "comment": "pangram",
          "input": "törkylempijävongahdus bcfqwxzå",
          "output": "⠞⠪⠗⠅⠽⠇⠑⠍⠏⠊⠚⠜⠧⠕⠝⠛⠁⠓⠙⠥⠎ ⠃⠉⠋⠟⠺⠭⠵⠡"
        }
      ]
    },
    
    {
      "data": [
        {
          "comment": "single digits",
          "input": "1 2 3 4 5 6 7 8 9 0",
          "output": "⠼⠁ ⠼⠃ ⠼⠉ ⠼⠙ ⠼⠑ ⠼⠋ ⠼⠛ ⠼⠓ ⠼⠊ ⠼⠚"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "generic symbols",
          "input": "= ! + - _ | \\ / [ ] { } ( ) ' \" # % ^ & * ~ , ? < > $ £ € ¢ @ ∞ √ ; : ° § . ·",
          "output": "⠶ ⠲ ⠖ ⠤ ⠬ ⠸ ⠩ ⠌ ⠷ ⠾ ⠫ ⠻ ⠦ ⠴ ⠐ ⠰ ⠼ ⠹ ⠬ ⠯ ⠔ ⠘ ⠂ ⠢ ⠣ ⠱ ⠮ ⠇ ⠈⠑ ⠈⠉ ⠈ ⠿ ⠩ ⠆ ⠒ ⠴ ⠼ ⠄ ⠄"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "dashes and hyphens, p. 42-43",
          "input": "Kari–Mäki: Leivot. Kari-Mäki, Maija. Jalkapallo-ottelu päättyi 2-3.",
          "output": "⠠⠅⠁⠗⠊ ⠤ ⠠⠍⠜⠅⠊⠒ ⠠⠇⠑⠊⠧⠕⠞⠄ ⠠⠅⠁⠗⠊⠤⠠⠍⠜⠅⠊⠂ ⠠⠍⠁⠊⠚⠁⠄ ⠠⠚⠁⠇⠅⠁⠏⠁⠇⠇⠕⠤⠕⠞⠞⠑⠇⠥ ⠏⠜⠜⠞⠞⠽⠊ ⠼⠃⠤⠼⠉⠄"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "degree sign and parenthesis, p. 44",
          "input": "Suora kulma on 90°. (Paitsi poikkeustapauksissa.)",
          "output": "⠠⠎⠥⠕⠗⠁ ⠅⠥⠇⠍⠁ ⠕⠝ ⠼⠊⠚ ⠴⠄ ⠦⠠⠏⠁⠊⠞⠎⠊ ⠏⠕⠊⠅⠅⠑⠥⠎⠞⠁⠏⠁⠥⠅⠎⠊⠎⠎⠁⠄⠴"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "common foreign language characters",
          "input": "é ß ü œ æ",
          "output": "⠿ ⠮ ⠳ ⠪ ⠜"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "numbers to text, different combinations",
          "input": "123abc abc123 abc123def 123abc456",
          "output": "⠼⠁⠃⠉⠰⠁⠃⠉ ⠁⠃⠉⠼⠁⠃⠉ ⠁⠃⠉⠼⠁⠃⠉⠰⠙⠑⠋ ⠼⠁⠃⠉⠰⠁⠃⠉⠼⠙⠑⠋"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "capitalizations, p. 36",
          "input": "Haluan korostaa HYVÄÄ käytöstä. OPM",
          "output": "⠠⠓⠁⠇⠥⠁⠝ ⠅⠕⠗⠕⠎⠞⠁⠁ ⠠⠠⠓⠽⠧⠜⠜ ⠅⠜⠽⠞⠪⠎⠞⠜⠄ ⠠⠠⠕⠏⠍"<commit_msg>add test coverage for apostrophe<commit_after>{
  "tables": [
    "unicode.dis",
    "fi.ctb"
  ],

  "tests": [
    {
      "data": [
        {
          "comment": "pangram",
          "input": "törkylempijävongahdus bcfqwxzå",
          "output": "⠞⠪⠗⠅⠽⠇⠑⠍⠏⠊⠚⠜⠧⠕⠝⠛⠁⠓⠙⠥⠎ ⠃⠉⠋⠟⠺⠭⠵⠡"
        }
      ]
    },
    
    {
      "data": [
        {
          "comment": "single digits",
          "input": "1 2 3 4 5 6 7 8 9 0",
          "output": "⠼⠁ ⠼⠃ ⠼⠉ ⠼⠙ ⠼⠑ ⠼⠋ ⠼⠛ ⠼⠓ ⠼⠊ ⠼⠚"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "generic symbols",
          "input": "= ! + - _ | \\ / [ ] { } ( ) ' \" # % ^ & * ~ , ? < > $ £ € ¢ @ ∞ √ ; : ° § . ·",
          "output": "⠶ ⠲ ⠖ ⠤ ⠬ ⠸ ⠩ ⠌ ⠷ ⠾ ⠫ ⠻ ⠦ ⠴ ⠐ ⠰ ⠼ ⠹ ⠬ ⠯ ⠔ ⠘ ⠂ ⠢ ⠣ ⠱ ⠮ ⠇ ⠈⠑ ⠈⠉ ⠈ ⠿ ⠩ ⠆ ⠒ ⠴ ⠼ ⠄ ⠄"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "dashes and hyphens, p. 42-43",
          "input": "Kari–Mäki: Leivot. Kari-Mäki, Maija. Jalkapallo-ottelu päättyi 2-3.",
          "output": "⠠⠅⠁⠗⠊ ⠤ ⠠⠍⠜⠅⠊⠒ ⠠⠇⠑⠊⠧⠕⠞⠄ ⠠⠅⠁⠗⠊⠤⠠⠍⠜⠅⠊⠂ ⠠⠍⠁⠊⠚⠁⠄ ⠠⠚⠁⠇⠅⠁⠏⠁⠇⠇⠕⠤⠕⠞⠞⠑⠇⠥ ⠏⠜⠜⠞⠞⠽⠊ ⠼⠃⠤⠼⠉⠄"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "degree sign and parenthesis, p. 44",
          "input": "Suora kulma on 90°. (Paitsi poikkeustapauksissa.)",
          "output": "⠠⠎⠥⠕⠗⠁ ⠅⠥⠇⠍⠁ ⠕⠝ ⠼⠊⠚ ⠴⠄ ⠦⠠⠏⠁⠊⠞⠎⠊ ⠏⠕⠊⠅⠅⠑⠥⠎⠞⠁⠏⠁⠥⠅⠎⠊⠎⠎⠁⠄⠴"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "Apostrophes, p. 44",
          "input": "Yht'ihanaisena",
          "output": "⠠⠽⠓⠞⠐⠊⠓⠁⠝⠁⠊⠎⠑⠝⠁"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "common foreign language characters",
          "input": "é ß ü œ æ",
          "output": "⠿ ⠮ ⠳ ⠪ ⠜"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "numbers to text, different combinations",
          "input": "123abc abc123 abc123def 123abc456",
          "output": "⠼⠁⠃⠉⠰⠁⠃⠉ ⠁⠃⠉⠼⠁⠃⠉ ⠁⠃⠉⠼⠁⠃⠉⠰⠙⠑⠋ ⠼⠁⠃⠉⠰⠁⠃⠉⠼⠙⠑⠋"
        }
      ]
    },

    {
      "data": [
        {
          "comment": "capitalizations, p. 36",
          "input": "Haluan korostaa HYVÄÄ käytöstä. OPM",
          "output": "⠠⠓⠁⠇⠥⠁⠝ ⠅⠕⠗⠕⠎⠞⠁⠁ ⠠⠠⠓⠽⠧⠜⠜ ⠅⠜⠽⠞⠪⠎⠞⠜⠄ ⠠⠠⠕⠏⠍"
{'fraction_non_alphanumeric': 0.2916767189384801, 'fraction_numerical': 0.01833534378769602, 'mean_word_length': 1.3772935779816513, 'pattern_counts': {'":': 72, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5259537', 'n_tokens_mistral': 3174, 'n_tokens_neox': 2967, 'n_words': 398}