<commit_before>import org.scalatest.{Matchers, FlatSpec}

class OcrTest extends FlatSpec with Matchers {
  it should "recognize zero" in {
    Ocr(List(" _ "
           , "| |"
           , "|_|"
           , "   ").mkString("\n")).convert should be ("0")
  }

  it should "recognize one" in {
    Ocr(List("   "
           , "  |"
           , "  |"
           , "   ").mkString("\n")).convert should be ("1")
  }

  it should "recognize two" in {
    Ocr(List(" _ "
           , " _|"
           , "|_ "
           , "   ").mkString("\n")).convert should be ("2")
  }

  it should "recognize three" in {
    Ocr(List(" _ "
           , " _|"
           , " _|"
           , "   ").mkString("\n")).convert should be ("3")
  }

  it should "recognize four" in {
    Ocr(List("   "
           , "|_|"
           , "  |"
           , "   ").mkString("\n")).convert should be ("4")
  }

  it should "recognize five" in {
    Ocr(List(" _ "
           , "|_ "
           , " _|"
           , "   ").mkString("\n")).convert should be ("5")
  }

  it should "recognize six" in {
    Ocr(List(" _ "
           , "|_ "
           , "|_|"
           , "   ").mkString("\n")).convert should be ("6")
  }

  it should "recognize seven" in {
    Ocr(List(" _ "
           , "  |"
           , "  |"
           , "   ").mkString("\n")).convert should be ("7")
  }

  it should "recognize eight" in {
    Ocr(List(" _ "
           , "|_|"
           , "|_|"
           , "   ").mkString("\n")).convert should be ("8")
  }

  it should "recognize nine" in {
    Ocr(List(" _ "
           , "|_|"
           , " _|"
           , "   ").mkString("\n")).convert should be ("9")
  }

  it should "recognize garble" in {
    Ocr(List("   "
           , "| |"
           , "| |"
           , "   ").mkString("\n")).convert should be ("?")
  }

  it should "recognize ten" in {
    Ocr(List("    _ "
           , "  || |"
           , "  ||_|"
           , "      ").mkString("\n")).convert should be ("10")
  }

  it should "recognize 110101100" in {
    Ocr(List("       _     _        _  _ "
           , "  |  || |  || |  |  || || |"
           , "  |  ||_|  ||_|  |  ||_||_|"
           , "                           ").mkString("\n")).convert should be ("110101100")
  }

  it should "recognize garbled multi digit ocr" in {
    Ocr(List("       _     _           _ "
           , "  |  || |  || |     || || |"
           , "  |  | _|  ||_|  |  ||_||_|"
           , "                           ").mkString("\n")).convert should be ("11?10?1?0")
  }

  it should "recognize 1234567890" in {
    Ocr(List("    _  _     _  _  _  _  _  _ "
           , "  | _| _||_||_ |_   ||_||_|| |"
           , "  ||_  _|  | _||_|  ||_| _||_|"
           , "                              ").mkString("\n")).convert should be ("1234567890")
  }

  it should "recognize 123,456,789" in {
    Ocr(List("    _  _ "
           , "  | _| _|"
           , "  ||_  _|"
           , "         "
           , "    _  _ "
           , "|_||_ |_ "
           , "  | _||_|"
           , "         "
           , " _  _  _ "
           , "  ||_||_|"
           , "  ||_| _|"
           , "         ").mkString("\n")).convert should be ("123,456,789")
  }
}
<commit_msg>Added pending keyword to ocr-numbers test<commit_after>import org.scalatest.{Matchers, FlatSpec}

class OcrTest extends FlatSpec with Matchers {
  it should "recognize zero" in {
    Ocr(List(" _ "
           , "| |"
           , "|_|"
           , "   ").mkString("\n")).convert should be ("0")
  }

  it should "recognize one" in {
    pending
    Ocr(List("   "
           , "  |"
           , "  |"
           , "   ").mkString("\n")).convert should be ("1")
  }

  it should "recognize two" in {
    pending
    Ocr(List(" _ "
           , " _|"
           , "|_ "
           , "   ").mkString("\n")).convert should be ("2")
  }

  it should "recognize three" in {
    pending
    Ocr(List(" _ "
           , " _|"
           , " _|"
           , "   ").mkString("\n")).convert should be ("3")
  }

  it should "recognize four" in {
    pending
    Ocr(List("   "
           , "|_|"
           , "  |"
           , "   ").mkString("\n")).convert should be ("4")
  }

  it should "recognize five" in {
    pending
    Ocr(List(" _ "
           , "|_ "
           , " _|"
           , "   ").mkString("\n")).convert should be ("5")
  }

  it should "recognize six" in {
    pending
    Ocr(List(" _ "
           , "|_ "
           , "|_|"
           , "   ").mkString("\n")).convert should be ("6")
  }

  it should "recognize seven" in {
    pending
    Ocr(List(" _ "
           , "  |"
           , "  |"
           , "   ").mkString("\n")).convert should be ("7")
  }

  it should "recognize eight" in {
    pending
    Ocr(List(" _ "
           , "|_|"
           , "|_|"
           , "   ").mkString("\n")).convert should be ("8")
  }

  it should "recognize nine" in {
    pending
    Ocr(List(" _ "
           , "|_|"
           , " _|"
           , "   ").mkString("\n")).convert should be ("9")
  }

  it should "recognize garble" in {
    pending
    Ocr(List("   "
           , "| |"
           , "| |"
           , "   ").mkString("\n")).convert should be ("?")
  }

  it should "recognize ten" in {
    pending
    Ocr(List("    _ "
           , "  || |"
           , "  ||_|"
           , "      ").mkString("\n")).convert should be ("10")
  }

  it should "recognize 110101100" in {
    pending
    Ocr(List("       _     _        _  _ "
           , "  |  || |  || |  |  || || |"
           , "  |  ||_|  ||_|  |  ||_||_|"
           , "                           ").mkString("\n")).convert should be ("110101100")
  }

  it should "recognize garbled multi digit ocr" in {
    pending
    Ocr(List("       _     _           _ "
           , "  |  || |  || |     || || |"
           , "  |  | _|  ||_|  |  ||_||_|"
           , "                           ").mkString("\n")).convert should be ("11?10?1?0")
  }

  it should "recognize 1234567890" in {
    pending
    Ocr(List("    _  _     _  _  _  _  _  _ "
           , "  | _| _||_||_ |_   ||_||_|| |"
           , "  ||_  _|  | _||_|  ||_| _||_|"
           , "                              ").mkString("\n")).convert should be ("1234567890")
  }

  it should "recognize 123,456,789" in {
    pending
    Ocr(List("    _  _ "
           , "  | _| _|"
           , "  ||_  _|"
           , "         "
           , "    _  _ "
           , "|_||_ |_ "
           , "  | _||_|"
           , "         "
           , " _  _  _ "
           , "  ||_||_|"
           , "  ||_| _|"
           , "         ").mkString("\n")).convert should be ("123,456,789")
  }
}

{'fraction_non_alphanumeric': 0.19816706730769232, 'fraction_numerical': 0.022235576923076924, 'mean_word_length': 1.1684039087947882, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1210321', 'n_tokens_mistral': 2529, 'n_tokens_neox': 2359, 'n_words': 362}