<commit_before>{-# LANGUAGE OverloadedStrings #-}
module Builder where

import Data.Char (ord)
import Data.Monoid (mconcat)

import qualified Data.Binary.Builder as Binary
import Criterion.Main
import qualified Data.ByteString.Lazy as LB
import qualified Data.ByteString as S
import Data.Text (Text)
import Data.Text.Encoding (encodeUtf8)

import qualified Text.Blaze.Builder.Core as Blaze
import qualified Text.Blaze.Builder.Utf8 as Blaze

main :: IO ()
main = defaultMain $ concat
    [ benchmark "[String]"
        (mconcat . concatMap (map $ Binary.singleton .  fromIntegral . ord))
        (mconcat . map Blaze.fromString)
        strings
    , benchmark "[S.ByteString]"
        (mconcat . map Binary.fromByteString)
        (mconcat . map Blaze.fromByteString)
        byteStrings
    , benchmark "[Text]"
        (mconcat . map (Binary.fromByteString . encodeUtf8))
        (mconcat . map Blaze.fromText)
        texts
    ]
  where
    benchmark name binaryF blazeF x =
        [ bench (name ++ " (Data.Binary builder)") $
            whnf (LB.length . Binary.toLazyByteString . binaryF) x
        , bench (name ++ " (blaze builder)") $
            whnf (LB.length . Blaze.toLazyByteString . blazeF) x
        ]

    strings :: [String]
    strings = replicate 10000 "<img>"
    {-# NOINLINE strings #-}

    byteStrings :: [S.ByteString]
    byteStrings = replicate 10000 "<img>"
    {-# NOINLINE byteStrings #-}

    texts :: [Text]
    texts = replicate 10000 "<img>"
    {-# NOINLINE texts #-}<commit_msg>Add benchmark timing byte writes<commit_after>{-# LANGUAGE OverloadedStrings #-}
module Main where

import Data.Char (ord)
import Data.Monoid (mconcat)
import Data.Word (Word8)

import qualified Data.Binary.Builder as Binary
import Criterion.Main
import qualified Data.ByteString.Lazy as LB
import qualified Data.ByteString as S
import Data.Text (Text)
import Data.Text.Encoding (encodeUtf8)

import qualified Text.Blaze.Builder.Core as Blaze
import qualified Text.Blaze.Builder.Utf8 as Blaze

main :: IO ()
main = defaultMain $ concat
    [ benchmark "[String]"
        (mconcat . concatMap (map $ Binary.singleton .  fromIntegral . ord))
        (mconcat . map Blaze.fromString)
        strings
    , benchmark "[S.ByteString]"
        (mconcat . map Binary.fromByteString)
        (mconcat . map Blaze.fromByteString)
        byteStrings
    , benchmark "[Text]"
        (mconcat . map (Binary.fromByteString . encodeUtf8))
        (mconcat . map Blaze.fromText)
        texts
    , benchmark "[Word8]"
        (mconcat . map Binary.singleton)
        (mconcat . map Blaze.singleton)
        word8s
    ]
  where
    benchmark name binaryF blazeF x =
        [ bench (name ++ " (Data.Binary builder)") $
            whnf (LB.length . Binary.toLazyByteString . binaryF) x
        , bench (name ++ " (blaze builder)") $
            whnf (LB.length . Blaze.toLazyByteString . blazeF) x
        ]

    strings :: [String]
    strings = replicate 10000 "<img>"
    {-# NOINLINE strings #-}

    byteStrings :: [S.ByteString]
    byteStrings = replicate 10000 "<img>"
    {-# NOINLINE byteStrings #-}

    texts :: [Text]
    texts = replicate 10000 "<img>"
    {-# NOINLINE texts #-}

    word8s :: [Word8]
    word8s = replicate 10000 $ fromIntegral $ ord 'a'
    {-# NOINLINE word8s #-}
{'fraction_non_alphanumeric': 0.11228813559322035, 'fraction_numerical': 0.014527845036319613, 'mean_word_length': 2.6559734513274336, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1836905', 'n_tokens_mistral': 1098, 'n_tokens_neox': 1009, 'n_words': 284}