<commit_before>
module Test.Bakers12.Tokenizer
    ( tokenizerTests
    ) where

import           Control.Exception (throw)
import qualified Data.Char as C
import qualified Data.List as L
import qualified Data.Text as T
import           Test.Framework (Test, testGroup)
import           Test.Framework.Providers.QuickCheck2 (testProperty)
import           Test.QuickCheck
import           Text.Bakers12.Tokenizer

instance Arbitrary T.Text where
    arbitrary =
        fmap T.pack . listOf1 $ suchThat (choose chrRange) C.isPrint
        where chrRange = (C.chr 0, C.chr maxChr)

              maxChr   :: Int
              maxChr   = (fromIntegral 2) ^ (fromIntegral 16)

    shrink = T.tails

-- This tokenizes some text and throws errors.
tokenize' :: T.Text -> [Token]
tokenize' input =
    case tokenize input of
        Right tokens -> tokens
        Left  err    -> throw err

-- tokenText must be normalized with no upper-case characters.
prop_normalized :: T.Text -> Bool
prop_normalized = L.all textIsLower . tokenize'
    where charIsLower c = (not $ C.isAlpha c) || (C.isLower c)
          textIsLower = T.all charIsLower . tokenText

-- The length of the tokens must equal the length of the raw text.
prop_tokenLength :: T.Text -> Bool
prop_tokenLength = L.all lengthIsEqualRawLength . tokenize'
    where lengthIsEqualRawLength t = tokenLength t == T.length (tokenRaw t)

-- The total length of the tokens equals the length of the raw input.
prop_totalLength :: T.Text -> Bool
prop_totalLength input = T.length input == total
    where total = L.sum . map tokenLength $ tokenize' input<commit_msg>Don't use shrinking for Text quick checks.<commit_after>
module Test.Bakers12.Tokenizer
    ( tokenizerTests
    ) where

import           Control.Exception (throw)
import qualified Data.Char as C
import qualified Data.List as L
import qualified Data.Text as T
import           Test.Framework (Test, testGroup)
import           Test.Framework.Providers.QuickCheck2 (testProperty)
import           Test.QuickCheck
import           Text.Bakers12.Tokenizer

instance Arbitrary T.Text where
    arbitrary =
        fmap T.pack . listOf1 $ suchThat (choose chrRange) C.isPrint
        where chrRange = (C.chr 0, C.chr maxChr)

              maxChr   :: Int
              maxChr   = (fromIntegral 2) ^ (fromIntegral 16)

    shrink = shrinkNothing

-- This tokenizes some text and throws errors.
tokenize' :: T.Text -> [Token]
tokenize' input =
    case tokenize input of
        Right tokens -> tokens
        Left  err    -> throw err

-- tokenText must be normalized with no upper-case characters.
prop_normalized :: T.Text -> Bool
prop_normalized = L.all textIsLower . tokenize'
    where charIsLower c = (not $ C.isAlpha c) || (C.isLower c)
          textIsLower = T.all charIsLower . tokenText

-- The length of the tokens must equal the length of the raw text.
prop_tokenLength :: T.Text -> Bool
prop_tokenLength = L.all lengthIsEqualRawLength . tokenize'
    where lengthIsEqualRawLength t = tokenLength t == T.length (tokenRaw t)

-- The total length of the tokens equals the length of the raw input.
prop_totalLength :: T.Text -> Bool
prop_totalLength input = T.length input == total
    where total = L.sum . map tokenLength $ tokenize' input
{'fraction_non_alphanumeric': 0.07824486038662166, 'fraction_numerical': 0.00613685179502915, 'mean_word_length': 2.760092272202999, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1986688', 'n_tokens_mistral': 957, 'n_tokens_neox': 928, 'n_words': 339}