<commit_before>-----------------------------------------------------------------------------
-- |
-- Module      : Data.Binary
-- Copyright   : <NAME>
-- License     : BSD3-style (see LICENSE)
-- 
-- Maintainer  : <NAME> <<EMAIL>>
-- Stability   : unstable
-- Portability : FFI + (currently) flexible instances
--
-- Binary serialisation of values to and from lazy ByteStrings.
--
-----------------------------------------------------------------------------

module Data.Binary (

    -- * The Binary class
      Binary(..)

    -- * The Get and Put monads
    , Get
    , Put

    -- * Useful helpers for writing instances
    , putWord8
    , getWord8

    -- * Binary serialisation
    , encode                    -- :: Binary a => a -> ByteString
    , decode                    -- :: Binary a => ByteString -> a

    -- * IO functions for serialisation
    , hEncode                   -- :: Binary a => Handle -> a -> IO ()
    , hDecode                   -- :: Binary a => Handle -> IO a
    , encodeFile                -- :: Binary a => FilePath -> a -> IO ()
    , decodeFile                -- :: Binary a => FilePath -> IO a

    ) where

import Data.Binary.Put
import Data.Binary.Get

import Control.Monad
import Foreign

import Data.ByteString.Lazy (ByteString)
import qualified Data.ByteString.Lazy as L

-- and needed for the instances:
import qualified Data.ByteString as B
import qualified Data.Map        as Map
import qualified Data.Set        as Set
import qualified Data.IntMap     as IntMap
import qualified Data.IntSet     as IntSet

import Data.Array.Unboxed
import Data.List (unfoldr)
import qualified Data.Tree as T
import qualified Data.Sequence as Seq

import System.IO

--
-- For the Integer instance
--
import Data.ByteString.Base (toForeignPtr,unsafePackAddress, memcpy)
import GHC.Num
import GHC.Base
import GHC.Prim
import GHC.Ptr (Ptr(..))
import GHC.IOBase (IO(..))

------------------------------------------------------------------------

-- | The @Binary@ class provides 'put' and 'get', methods to encode and
-- decode a value to a lazy bytestring.
--
-- New instances for binary should have the following property:
--
-- > get . put == id
--
-- A range of instances are provided for basic Haskell types. To
-- serialise a custom type, an instance of Binary for that type is
-- required. For example, suppose we have a data structure:
--
-- > data Exp = IntE Int
-- >          | OpE  String Exp Exp
-- >    deriving Show
--
-- We can encode values of this type into bytestrings using the
-- following instance, which proceeds by recursively breaking down the
-- structure to serialise:
--
-- > instance Binary Exp where
-- >       put (IntE i)          = do put (0 :: Word8)
-- >                                  put i
-- >       put (OpE s e1 e2)     = do put (1 :: Word8)
-- >                                  put s
-- >                                  put e1
-- >                                  put e2
-- > 
-- >       get = do t <- get :: Get Word8
-- >                case t of
-- >                     0 -> do i <- get
-- >                             return (IntE i)
-- >                     1 -> do s  <- get
-- >                             e1 <- get
-- >                             e2 <- get
-- >                             return (OpE s e1 e2)
--
-- Note how we write an initial tag byte to indicate each variant of the
-- data type.
--
-- To serialise this to a bytestring, we use 'encode', which packs the
-- data structure into a binary format, in a lazy bytestring
--
-- > > let e = OpE "*" (IntE 7) (OpE "/" (IntE 4) (IntE 2))
-- > > let v = encode e
--
-- Where 'v' is a binary encoded data structure. To reconstruct the
-- original data, we use 'decode'
--
-- > > decode v :: Exp
-- > OpE "*" (IntE 7) (OpE "/" (IntE 4) (IntE 2))
--
-- The lazy ByteString that results from 'encode' can be written to
-- disk, and read from disk using Data.ByteString.Lazy IO functions,
-- such as hPutStr or writeFile:
--
-- > > writeFile "/tmp/exp.txt" (encode e)
--
-- And read back with:
--
-- > > readFile "/tmp/exp.txt" >>= return . decode :: IO Exp
-- > OpE "*" (IntE 7) (OpE "/" (IntE 4) (IntE 2))
--
-- We can also directly serialise a value to and from a Handle, or a file:
-- 
-- > > v <- decodeFile  "/tmp/exp.txt" :: IO Exp
-- > OpE "*" (IntE 7) (OpE "/" (IntE 4) (IntE 2))
--
-- And write a value to disk
--
-- > > encodeFile "/tmp/a.txt" v
--
class Binary t where
    -- | Encode a value in the Put monad.
    put :: t -> Put ()
    -- | Decode a value in the Get monad
    get :: Get t

------------------------------------------------------------------------
-- Wrappers to run the underlying monad

-- | Encode a value using binary serialisation to a lazy ByteString.
--
encode :: Binary a => a -> ByteString
encode = runPut . put

-- | Decode a value from a lazy ByteString, reconstructing the original structure.
--
decode :: Binary a => ByteString -> a
decode = runGet get

------------------------------------------------------------------------
-- Convenience IO operations

-- | Serialise a value to a file
--
-- This is just a convenience function, it's defined simply as:
--
-- > encodeFile f = B.writeFile f . encode
--
-- So for example if you wanted to compress as well, you could use:
--
-- > B.writeFile f . compress . encode
--
encodeFile :: Binary a => FilePath -> a -> IO ()
encodeFile f v = L.writeFile f (encode v)

-- | Reconstruct a value previously written to a file
--
-- This is just a convenience function, it's defined simply as:
--
-- > decodeFile f = return . decode =<< B.readFile f
--
-- So for example if you wanted to decompress as well, you could use:
--
-- > return . decode . decompress =<< B.readFile f
--
decodeFile :: Binary a => FilePath -> IO a
decodeFile f = liftM decode (L.readFile f)

------------------------------------------------------------------------
-- Simple instances

instance Binary () where
    put ()  = return ()
    get     = return ()

instance Binary Ordering where
    put = putWord8 . fromIntegral . fromEnum
    get = liftM (toEnum . fromIntegral) getWord8

------------------------------------------------------------------------
-- Words and Ints

instance Binary Word8 where
    put     = putWord8
    get     = getWord8

instance Binary Bool where
    put     = putWord8 . fromIntegral . fromEnum
    get     = liftM (toEnum . fromIntegral) getWord8<commit_msg>not that encodeFile/decodeFile are lazy<commit_after>-----------------------------------------------------------------------------
-- |
-- Module      : Data.Binary
-- Copyright   : <NAME>
-- License     : BSD3-style (see LICENSE)
-- 
-- Maintainer  : <NAME> <<EMAIL>>
-- Stability   : unstable
-- Portability : FFI + (currently) flexible instances
--
-- Binary serialisation of values to and from lazy ByteStrings.
--
-----------------------------------------------------------------------------

module Data.Binary (

    -- * The Binary class
      Binary(..)

    -- * The Get and Put monads
    , Get
    , Put

    -- * Useful helpers for writing instances
    , putWord8
    , getWord8

    -- * Binary serialisation
    , encode                    -- :: Binary a => a -> ByteString
    , decode                    -- :: Binary a => ByteString -> a

    -- * IO functions for serialisation
    , hEncode                   -- :: Binary a => Handle -> a -> IO ()
    , hDecode                   -- :: Binary a => Handle -> IO a
    , encodeFile                -- :: Binary a => FilePath -> a -> IO ()
    , decodeFile                -- :: Binary a => FilePath -> IO a

    ) where

import Data.Binary.Put
import Data.Binary.Get

import Control.Monad
import Foreign

import Data.ByteString.Lazy (ByteString)
import qualified Data.ByteString.Lazy as L

-- and needed for the instances:
import qualified Data.ByteString as B
import qualified Data.Map        as Map
import qualified Data.Set        as Set
import qualified Data.IntMap     as IntMap
import qualified Data.IntSet     as IntSet

import Data.Array.Unboxed
import Data.List (unfoldr)
import qualified Data.Tree as T
import qualified Data.Sequence as Seq

import System.IO

--
-- For the Integer instance
--
import Data.ByteString.Base (toForeignPtr,unsafePackAddress, memcpy)
import GHC.Num
import GHC.Base
import GHC.Prim
import GHC.Ptr (Ptr(..))
import GHC.IOBase (IO(..))

------------------------------------------------------------------------

-- | The @Binary@ class provides 'put' and 'get', methods to encode and
-- decode a value to a lazy bytestring.
--
-- New instances for binary should have the following property:
--
-- > get . put == id
--
-- A range of instances are provided for basic Haskell types. To
-- serialise a custom type, an instance of Binary for that type is
-- required. For example, suppose we have a data structure:
--
-- > data Exp = IntE Int
-- >          | OpE  String Exp Exp
-- >    deriving Show
--
-- We can encode values of this type into bytestrings using the
-- following instance, which proceeds by recursively breaking down the
-- structure to serialise:
--
-- > instance Binary Exp where
-- >       put (IntE i)          = do put (0 :: Word8)
-- >                                  put i
-- >       put (OpE s e1 e2)     = do put (1 :: Word8)
-- >                                  put s
-- >                                  put e1
-- >                                  put e2
-- > 
-- >       get = do t <- get :: Get Word8
-- >                case t of
-- >                     0 -> do i <- get
-- >                             return (IntE i)
-- >                     1 -> do s  <- get
-- >                             e1 <- get
-- >                             e2 <- get
-- >                             return (OpE s e1 e2)
--
-- Note how we write an initial tag byte to indicate each variant of the
-- data type.
--
-- To serialise this to a bytestring, we use 'encode', which packs the
-- data structure into a binary format, in a lazy bytestring
--
-- > > let e = OpE "*" (IntE 7) (OpE "/" (IntE 4) (IntE 2))
-- > > let v = encode e
--
-- Where 'v' is a binary encoded data structure. To reconstruct the
-- original data, we use 'decode'
--
-- > > decode v :: Exp
-- > OpE "*" (IntE 7) (OpE "/" (IntE 4) (IntE 2))
--
-- The lazy ByteString that results from 'encode' can be written to
-- disk, and read from disk using Data.ByteString.Lazy IO functions,
-- such as hPutStr or writeFile:
--
-- > > writeFile "/tmp/exp.txt" (encode e)
--
-- And read back with:
--
-- > > readFile "/tmp/exp.txt" >>= return . decode :: IO Exp
-- > OpE "*" (IntE 7) (OpE "/" (IntE 4) (IntE 2))
--
-- We can also directly serialise a value to and from a Handle, or a file:
-- 
-- > > v <- decodeFile  "/tmp/exp.txt" :: IO Exp
-- > OpE "*" (IntE 7) (OpE "/" (IntE 4) (IntE 2))
--
-- And write a value to disk
--
-- > > encodeFile "/tmp/a.txt" v
--
class Binary t where
    -- | Encode a value in the Put monad.
    put :: t -> Put ()
    -- | Decode a value in the Get monad
    get :: Get t

------------------------------------------------------------------------
-- Wrappers to run the underlying monad

-- | Encode a value using binary serialisation to a lazy ByteString.
--
encode :: Binary a => a -> ByteString
encode = runPut . put

-- | Decode a value from a lazy ByteString, reconstructing the original structure.
--
decode :: Binary a => ByteString -> a
decode = runGet get

------------------------------------------------------------------------
-- Convenience IO operations

-- | Serialise a value to a file
encodeFile :: Binary a => FilePath -> a -> IO ()
encodeFile f v = L.writeFile f (encode v)

-- | Reconstruct a value previously written to a file
decodeFile :: Binary a => FilePath -> IO a
decodeFile f = liftM decode (L.readFile f)

------------------------------------------------------------------------
-- Simple instances

instance Binary () where
    put ()  = return ()
    get     = return ()

instance Binary Ordering where
    put = putWord8 . fromIntegral . fromEnum
    get = liftM (toEnum . fromIntegral) getWord8

------------------------------------------------------------------------
-- Words and Ints

instance Binary Word8 where
    put     = putWord8
    get     = getWord8

instance Binary Bool where
    put     = putWord8 . fromIntegral . fromEnum
    get     = liftM (toEnum . fromIntegral) getWord8
{'fraction_non_alphanumeric': 0.19430472172643193, 'fraction_numerical': 0.006003569690085997, 'mean_word_length': 1.4862847922549416, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 145, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7615615', 'n_tokens_mistral': 3481, 'n_tokens_neox': 3342, 'n_words': 1353}