<commit_before>module Network.Wai.Source
    (
    -- * Conversions
      toEnumerator
    , toLBS
    ) where

import Network.Wai
import qualified Data.ByteString.Lazy as L
import System.IO.Unsafe (unsafeInterleaveIO)

-- | This function safely converts a 'Source' (where you pull data) to an
-- 'Enumerator' (which pushes the data to you). There should be no significant
-- performance impact from its use, and it uses no unsafe functions.
toEnumerator :: Source -> Enumerator
toEnumerator source0 = Enumerator $ helper source0 where
    helper source iter a = do
        next <- runSource source
        case next of
            Nothing -> return $ Right a
            Just (bs, source') -> do
                res <- iter a bs
                case res of
                    Left a' -> return $ Left a'
                    Right a' -> helper source' iter a'

-- | Uses lazy I\/O (via 'unsafeInterleaveIO') to provide a lazy interface to
-- the given 'Source'. Normal lazy I\/O warnings apply.
toLBS :: Source -> IO L.ByteString
toLBS source0 = L.fromChunks `fmap` helper source0 where
    helper source = unsafeInterleaveIO $ do
        next <- runSource source
        case next of
            Nothing -> return []
            Just (bs, source') -> do
                rest <- helper source'
                return $ bs : rest<commit_msg>Added LBS -> Source conversion<commit_after>module Network.Wai.Source
    (
    -- * Conversions
      toEnumerator
    , toLBS
    , fromLBS
    ) where

import Network.Wai
import qualified Data.ByteString.Lazy as L
import System.IO.Unsafe (unsafeInterleaveIO)

-- | This function safely converts a 'Source' (where you pull data) to an
-- 'Enumerator' (which pushes the data to you). There should be no significant
-- performance impact from its use, and it uses no unsafe functions.
toEnumerator :: Source -> Enumerator
toEnumerator source0 = Enumerator $ helper source0 where
    helper source iter a = do
        next <- runSource source
        case next of
            Nothing -> return $ Right a
            Just (bs, source') -> do
                res <- iter a bs
                case res of
                    Left a' -> return $ Left a'
                    Right a' -> helper source' iter a'

-- | Uses lazy I\/O (via 'unsafeInterleaveIO') to provide a lazy interface to
-- the given 'Source'. Normal lazy I\/O warnings apply.
toLBS :: Source -> IO L.ByteString
toLBS source0 = L.fromChunks `fmap` helper source0 where
    helper source = unsafeInterleaveIO $ do
        next <- runSource source
        case next of
            Nothing -> return []
            Just (bs, source') -> do
                rest <- helper source'
                return $ bs : rest

-- | Convert a lazy bytestring to a 'Source'. This operation does not request lazy I\/O.
fromLBS :: L.ByteString -> Source
fromLBS =
    go . L.toChunks
  where
    go [] = Source $ return Nothing
    go (x:xs) = Source $ return $ Just (x, go xs)
{'fraction_non_alphanumeric': 0.08611955420466058, 'fraction_numerical': 0.002701789935832489, 'mean_word_length': 2.059917355371901, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3474237', 'n_tokens_mistral': 823, 'n_tokens_neox': 797, 'n_words': 334}