<commit_before>module Prose.Types (CodePoint,Grapheme,Prose(..))
    where

import Data.Text (Text)
import Data.ByteString (ByteString)

-- | Unicode code point
type CodePoint = Char

-- | Grapheme cluster
type Grapheme = [Char]


data Prose = T T.Text             -- ^ Text's utf-16 data in NFC
           -- | U BS.ByteString       -- ^ ByteString with utf-8 in NFC

           -- ^ Well…ideally, change Text’s internal represetation to utf-8
<commit_msg>P.Types: Show, IsString<commit_after>module Prose.Types (CodePoint,Grapheme,Prose(..))
    where

import qualified Data.Text as T
import qualified Data.ByteString as BS
import Data.String
import Data.Monoid

-- | Unicode code point
type CodePoint = Char

-- | Grapheme cluster
type Grapheme = [Char]


data Prose = T T.Text             -- ^ Text's utf-16 data in NFC
           -- | U BS.ByteString       -- ^ ByteString with utf-8 in NFC

           -- ^ Well…ideally, change Text’s internal represetation to utf-8


-- TODO: NF

instance Show Prose where
    show (T txt) = "\"" <> T.unpack txt <> "\"" -- XXX only printable

instance IsString Prose where
    fromString = T . T.pack


{'fraction_non_alphanumeric': 0.10926573426573427, 'fraction_numerical': 0.006993006993006993, 'mean_word_length': 2.4908536585365852, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6459734', 'n_tokens_mistral': 380, 'n_tokens_neox': 360, 'n_words': 119}