<commit_before>-- | Generic stream manipulations

{-# LANGUAGE BangPatterns       #-}
{-# LANGUAGE DeriveDataTypeable #-}
{-# LANGUAGE RankNTypes         #-}

module System.IO.Streams.Combinators
 ( -- * Folds
   inputFoldM
 , outputFoldM
 , fold
 , foldM
 , any
 , all
 , maximum
 , minimum

   -- * Unfolds
 , unfoldM

   -- * Maps
 , map
 , mapM
 , mapM_
 , mapMaybe
 , contramap
 , contramapM
 , contramapM_
 , contramapMaybe

   -- * Filter
 , filter
 , filterM
 , filterOutput
 , filterOutputM

   -- * Takes and drops
 , give
 , take
 , drop
 , ignore

   -- * Zip and unzip
 , zip
 , zipWith
 , zipWithM
 , unzip

   -- * Utility
 , intersperse
 , skipToEof
 , ignoreEof
 , atEndOfInput
 , atEndOfOutput
 ) where

------------------------------------------------------------------------------
import           Control.Concurrent.MVar    (newMVar, withMVar)
import           Control.Monad              (liftM, void, when)
import           Control.Monad.IO.Class     (liftIO)
import           Data.Int                   (Int64)
import           Data.IORef                 (IORef, atomicModifyIORef, modifyIORef, newIORef, readIORef, writeIORef)
import           Data.Maybe                 (isJust)
import           Prelude                    hiding (all, any, drop, filter, map, mapM, mapM_, maximum, minimum, read, take, unzip, zip, zipWith)
------------------------------------------------------------------------------
import           System.IO.Streams.Internal (InputStream (..), OutputStream (..), fromGenerator, makeInputStream, makeOutputStream, read, unRead, write, yield)


------------------------------------------------------------------------------
-- | A side-effecting fold over an 'OutputStream', as a stream transformer.
--
-- The IO action returned by 'outputFoldM' can be used to fetch and reset the updated
-- seed value. Example:
--
-- @
-- ghci> is <- Streams.'System.IO.Streams.List.fromList' [1, 2, 3::Int]
-- ghci> (os, getList) <- Streams.'System.IO.Streams.List.listOutputStream'
-- ghci> (os', getSeed) \<- Streams.'outputFoldM' (\\x y -> return (x+y)) 0 os
-- ghci> Streams.'System.IO.Streams.connect' is os'
-- ghci> getList
-- [1,2,3]
-- ghci> getSeed
-- 6
-- @
outputFoldM :: (a -> b -> IO a)           -- ^ fold function
            -> a                          -- ^ initial seed
            -> OutputStream b             -- ^ output stream
            -> IO (OutputStream b, IO a)  -- ^ returns a new stream as well as
                                          -- an IO action to fetch and reset the
                                          --  updated seed value.
outputFoldM f initial stream = do
    ref <- newIORef initial
    os  <- makeOutputStream (wr ref)
    return (os, fetch ref)

  where
    wr _ Nothing       = write Nothing stream
    wr ref mb@(Just x) = do
        !z  <- readIORef ref
        !z' <- f z x
        writeIORef ref z'
        write mb stream

    fetch ref = atomicModifyIORef ref $ \x -> (initial, x)


------------------------------------------------------------------------------
-- | A side-effecting fold over an 'InputStream', as a stream transformer.
--
-- The IO action returned by 'inputFoldM' can be used to fetch and reset the updated seed
-- value. Example:
--
-- @
-- ghci> is <- Streams.'System.IO.Streams.List.fromList' [1, 2, 3::Int]
-- ghci> (is', getSeed) \<- Streams.'inputFoldM' (\\x y -> return (x+y)) 0 is
-- ghci> Streams.'System.IO.Streams.List.toList' is'
-- [1,2,3]
-- ghci> getSeed
-- 6
-- @
inputFoldM :: (a -> b -> IO a)          -- ^ fold function
           -> a                         -- ^ initial seed
           -> InputStream b             -- ^ input stream
           -> IO (InputStream b, IO a)  -- ^ returns a new stream as well as an
                                        -- IO action to fetch and reset the
                                        -- updated seed value.
inputFoldM f initial stream = do
    ref <- newIORef initial
    is  <- makeInputStream (rd ref)
    return (is, fetch ref)

  where
    twiddle _ Nothing = return Nothing

    twiddle ref mb@(Just x) = do
        !z  <- readIORef ref
        !z' <- f z x
        writeIORef ref z'
        return mb

    rd ref = read stream >>= twiddle ref

    fetch ref = atomicModifyIORef ref $ \x -> (initial, x)


------------------------------------------------------------------------------
-- | A left fold over an input stream. The input stream is fully consumed. See
-- 'Prelude.foldl'.
--
-- Example:
--
-- @
-- ghci> Streams.'System.IO.Streams.fromList' [1..10] >>= Streams.'fold' (+) 0
-- 55
-- @
fold :: (s -> a -> s)       -- ^ fold function
     -> s                   -- ^ initial seed
     -> InputStream a       -- ^ input stream
     -> IO s
fold f seed stream = go seed
  where
    go !s = read stream >>= maybe (return s) (go . f s)


------------------------------------------------------------------------------
-- | A side-effecting left fold over an input stream. The input stream is fully
-- consumed. See 'Prelude.foldl'.
--
-- Example:
--
-- @
-- ghci> Streams.'System.IO.Streams.fromList' [1..10] >>= Streams.'foldM' (\x y -> 'return' (x + y)) 0
-- 55
-- @
foldM :: (s -> a -> IO s)       -- ^ fold function
      -> s                      -- ^ initial seed
      -> InputStream a          -- ^ input stream
      -> IO s
foldM f seed stream = go seed
  where
    go !s = read stream >>= maybe (return s) ((go =<<) . f s)


------------------------------------------------------------------------------
-- | @any predicate stream@ returns 'True' if any element in @stream@ matches
-- the predicate.
--
-- 'any' consumes as few elements as possible, ending consumption if an element<commit_msg>Add fold functions for so-called beautiful folding<commit_after>-- | Generic stream manipulations

{-# LANGUAGE BangPatterns       #-}
{-# LANGUAGE DeriveDataTypeable #-}
{-# LANGUAGE RankNTypes         #-}

module System.IO.Streams.Combinators
 ( -- * Folds
   inputFoldM
 , outputFoldM
 , fold
 , foldM
 , fold_
 , foldM_
 , any
 , all
 , maximum
 , minimum

   -- * Unfolds
 , unfoldM

   -- * Maps
 , map
 , mapM
 , mapM_
 , mapMaybe
 , contramap
 , contramapM
 , contramapM_
 , contramapMaybe

   -- * Filter
 , filter
 , filterM
 , filterOutput
 , filterOutputM

   -- * Takes and drops
 , give
 , take
 , drop
 , ignore

   -- * Zip and unzip
 , zip
 , zipWith
 , zipWithM
 , unzip

   -- * Utility
 , intersperse
 , skipToEof
 , ignoreEof
 , atEndOfInput
 , atEndOfOutput
 ) where

------------------------------------------------------------------------------
import           Control.Concurrent.MVar    (newMVar, withMVar)
import           Control.Monad              (liftM, void, when)
import           Control.Monad.IO.Class     (liftIO)
import           Data.Int                   (Int64)
import           Data.IORef                 (IORef, atomicModifyIORef, modifyIORef, newIORef, readIORef, writeIORef)
import           Data.Maybe                 (isJust)
import           Prelude                    hiding (all, any, drop, filter, map, mapM, mapM_, maximum, minimum, read, take, unzip, zip, zipWith)
------------------------------------------------------------------------------
import           System.IO.Streams.Internal (InputStream (..), OutputStream (..), fromGenerator, makeInputStream, makeOutputStream, read, unRead, write, yield)


------------------------------------------------------------------------------
-- | A side-effecting fold over an 'OutputStream', as a stream transformer.
--
-- The IO action returned by 'outputFoldM' can be used to fetch and reset the updated
-- seed value. Example:
--
-- @
-- ghci> is <- Streams.'System.IO.Streams.List.fromList' [1, 2, 3::Int]
-- ghci> (os, getList) <- Streams.'System.IO.Streams.List.listOutputStream'
-- ghci> (os', getSeed) \<- Streams.'outputFoldM' (\\x y -> return (x+y)) 0 os
-- ghci> Streams.'System.IO.Streams.connect' is os'
-- ghci> getList
-- [1,2,3]
-- ghci> getSeed
-- 6
-- @
outputFoldM :: (a -> b -> IO a)           -- ^ fold function
            -> a                          -- ^ initial seed
            -> OutputStream b             -- ^ output stream
            -> IO (OutputStream b, IO a)  -- ^ returns a new stream as well as
                                          -- an IO action to fetch and reset the
                                          --  updated seed value.
outputFoldM f initial stream = do
    ref <- newIORef initial
    os  <- makeOutputStream (wr ref)
    return (os, fetch ref)

  where
    wr _ Nothing       = write Nothing stream
    wr ref mb@(Just x) = do
        !z  <- readIORef ref
        !z' <- f z x
        writeIORef ref z'
        write mb stream

    fetch ref = atomicModifyIORef ref $ \x -> (initial, x)


------------------------------------------------------------------------------
-- | A side-effecting fold over an 'InputStream', as a stream transformer.
--
-- The IO action returned by 'inputFoldM' can be used to fetch and reset the updated seed
-- value. Example:
--
-- @
-- ghci> is <- Streams.'System.IO.Streams.List.fromList' [1, 2, 3::Int]
-- ghci> (is', getSeed) \<- Streams.'inputFoldM' (\\x y -> return (x+y)) 0 is
-- ghci> Streams.'System.IO.Streams.List.toList' is'
-- [1,2,3]
-- ghci> getSeed
-- 6
-- @
inputFoldM :: (a -> b -> IO a)          -- ^ fold function
           -> a                         -- ^ initial seed
           -> InputStream b             -- ^ input stream
           -> IO (InputStream b, IO a)  -- ^ returns a new stream as well as an
                                        -- IO action to fetch and reset the
                                        -- updated seed value.
inputFoldM f initial stream = do
    ref <- newIORef initial
    is  <- makeInputStream (rd ref)
    return (is, fetch ref)

  where
    twiddle _ Nothing = return Nothing

    twiddle ref mb@(Just x) = do
        !z  <- readIORef ref
        !z' <- f z x
        writeIORef ref z'
        return mb

    rd ref = read stream >>= twiddle ref

    fetch ref = atomicModifyIORef ref $ \x -> (initial, x)


------------------------------------------------------------------------------
-- | A left fold over an input stream. The input stream is fully consumed. See
-- 'Prelude.foldl'.
--
-- Example:
--
-- @
-- ghci> Streams.'System.IO.Streams.fromList' [1..10] >>= Streams.'fold' (+) 0
-- 55
-- @
fold :: (s -> a -> s)       -- ^ fold function
     -> s                   -- ^ initial seed
     -> InputStream a       -- ^ input stream
     -> IO s
fold f seed stream = go seed
  where
    go !s = read stream >>= maybe (return s) (go . f s)


------------------------------------------------------------------------------
-- | A side-effecting left fold over an input stream. The input stream is fully
-- consumed. See 'Prelude.foldl'.
--
-- Example:
--
-- @
-- ghci> Streams.'System.IO.Streams.fromList' [1..10] >>= Streams.'foldM' (\x y -> 'return' (x + y)) 0
-- 55
-- @
foldM :: (s -> a -> IO s)       -- ^ fold function
      -> s                      -- ^ initial seed
      -> InputStream a          -- ^ input stream
      -> IO s
foldM f seed stream = go seed
  where
    go !s = read stream >>= maybe (return s) ((go =<<) . f s)


------------------------------------------------------------------------------
{- | A variant of 'System.IO.Streams.fold' suitable for use with composable folds
     from \'beautiful folding\' libraries like <http://hackage.haskell.org/package/foldl the foldl library>
     In particular, we have

>  Control.Foldl.purely fold_ :: Control.Fold.Fold a b -> InputStream a -> IO b
-}  

fold_ :: (x -> a -> x)    -- ^ accumulator update function
      -> x                -- ^ initial seed
      -> (x -> s)         -- ^ recover folded value
      -> InputStream a    -- ^ input stream
      -> IO s
fold_ op seed done stream = liftM done (go seed)
   where 
     go !s = read stream >>= maybe (return s) (go . op s)

------------------------------------------------------------------------------
{- | A variant of 'System.IO.Streams.foldM' suitable for use with composable folds
     from \'beautiful folding\' libraries like <http://hackage.haskell.org/package/foldl the foldl library>
     In particular, we have

>  Control.Foldl.impurely foldM_ :: Control.Fold.FoldM IO a b -> InputStream a -> IO b
-}  
foldM_ :: (x -> a -> IO x)   -- ^ accumulator update action
       -> IO x               -- ^ initial seed
       -> (x -> IO s)        -- ^ recover folded value
       -> InputStream a      -- ^ input stream
       -> IO s
foldM_ f seed done stream = seed >>= go 
  where
    go !x = read stream >>= maybe (done x) ((go =<<) . f x)

    -- > :t L.impurely foldM_
    -- L.impurely foldM_ :: FoldM IO a b -> InputStream a -> IO b



------------------------------------------------------------------------------
-- | @any predicate stream@ returns 'True' if any element in @stream@ matches
-- the predicate.
--
-- 'any' consumes as few elements as possible, ending consumption if an element
{'fraction_non_alphanumeric': 0.21352121352121353, 'fraction_numerical': 0.00462000462000462, 'mean_word_length': 1.3666180758017492, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 128, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2747723', 'n_tokens_mistral': 3832, 'n_tokens_neox': 3654, 'n_words': 1234}