<commit_before>-- | This module implements encoding and decoding of CSV data. The
-- implementation is RFC 4180 compliant, with the following
-- extensions:
--
--  * Empty lines are ignored.
--
--  * Non-escaped fields may contain any characters except
--    double-quotes, commas, carriage returns, and newlines.
--
--  * Escaped fields may contain any characters (but double-quotes
--    need to be escaped).
module Data.Csv
    (
    -- * Usage example
    -- $example

    -- * Treating CSV data as opaque byte strings
    -- $generic-processing

    -- * Custom type conversions
    -- $customtypeconversions

    -- * Encoding and decoding
    -- $encoding
      HasHeader(..)
    , decode
    , decodeByName
    , encode
    , encodeByName

    -- ** Encoding and decoding options
    -- $options
    , DecodeOptions(..)
    , defaultDecodeOptions
    , decodeWith
    , decodeByNameWith
    , EncodeOptions(..)
    , defaultEncodeOptions
    , encodeWith
    , encodeByNameWith

    -- * Core CSV types
    , Csv
    , Record
    , Field
    , Header
    , Name
    , NamedRecord

    -- * Type conversion
    -- $typeconversion

    -- ** Index-based record conversion
    -- $indexbased
    , FromRecord(..)
    , Parser
    , runParser
    , index
    , (.!)
    , unsafeIndex
    , ToRecord(..)
    , record
    , Only(..)

    -- ** Name-based record conversion
    -- $namebased
    , FromNamedRecord(..)
    , lookup
    , (.:)
    , ToNamedRecord(..)
    , namedRecord
    , namedField
    , (.=)

    -- ** Field conversion
    -- $fieldconversion
    , FromField(..)
    , ToField(..)
    ) where

import Prelude hiding (lookup)

import Data.Csv.Conversion
import Data.Csv.Encoding
import Data.Csv.Types

-- $example
--
-- A short encoding usage example:
--
-- > >>> encode [("John" :: Text, 27), ("Jane", 28)]
-- > Chunk "John,27\r\nJane,28\r\n" Empty
--
-- Since string literals are overloaded we have to supply a type
-- signature as the compiler couldn't deduce which string type (i.e.
-- 'String' or 'Text') we want to use. In most cases type inference
-- will infer the type from the context and you can omit type
-- signatures.
--
-- A short decoding usage example:
--
-- > >>> decode NoHeader "John,27\r\nJane,28\r\n" :: Either String (Vector (Text, Int))
-- > Right (fromList [("John",27),("Jane",28)])
--
-- We pass 'NoHeader' as the first argument to indicate that the CSV
-- input data isn't preceded by a header.
--
-- In practice, the return type of 'decode' rarely needs to be given,
-- as it can often be inferred from the context.

-- $generic-processing
--
-- Sometimes you might want to work with a CSV file which contents is
-- unknown to you. For example, you might want remove the second
-- column of a file without knowing anything about its content. To
-- parse a CSV file to a generic representation, just convert each
-- record to a @'Vector' 'ByteString'@ value, like so:
--
-- > decode NoHeader "John,27\r\nJane,28\r\n" :: Either String (Vector (Vector ByteString))
-- > Right (fromList [fromList ["John","27"],fromList ["Jane","28"]])
--
-- As the example output above shows, all the fields are returned as
-- uninterpreted 'ByteString' values.

-- $customtypeconversions
--
-- Most of the time the existing 'FromField' and 'ToField' instances
-- do what you want. However, if you need to parse a different format
-- (e.g. hex) but use a type (e.g. 'Int') for which there's already a
-- 'FromField' instance, you need to use a @newtype@. Example:
--
-- > newtype Hex = Hex Int
-- >
-- > parseHex :: ByteString -> Parser Int
-- > parseHex = ...
-- >
-- > instance FromField Hex where
-- >     parseField s = Hex <$> parseHex s
--
-- Other than giving an explicit type signature, you can pattern match
-- on the @newtype@ constructor to indicate which type conversion you
-- want to have the library use:
--
-- > case decode NoHeader "0xff,0xaa\r\n0x11,0x22\r\n" of
-- >     Left err -> putStrLn err
-- >     Right v  -> forM_ v $ \ (Hex val1, Hex val2) ->
-- >         print (val1, val2)
--
-- You can use the unit type, @()@, to ignore a column. The
-- 'parseField' method for @()@ doesn't look at the 'Field' and thus
-- always decodes successfully. Note that it lacks a corresponding
-- 'ToField' instance. Example:
--
-- > case decode NoHeader "foo,1\r\nbar,22" of
-- >     Left  err -> putStrLn err
-- >     Right v   -> forM_ v $ \ ((), i) -> print (i :: Int)

-- $encoding
--
-- Encoding and decoding is a two step process. To encode a value, it
-- is first converted to a generic representation, using either
-- 'ToRecord' or 'ToNamedRecord'. The generic representation is then<commit_msg>More examples of custom type conversion<commit_after>-- | This module implements encoding and decoding of CSV data. The
-- implementation is RFC 4180 compliant, with the following
-- extensions:
--
--  * Empty lines are ignored.
--
--  * Non-escaped fields may contain any characters except
--    double-quotes, commas, carriage returns, and newlines.
--
--  * Escaped fields may contain any characters (but double-quotes
--    need to be escaped).
module Data.Csv
    (
    -- * Usage example
    -- $example

    -- * Treating CSV data as opaque byte strings
    -- $generic-processing

    -- * Custom type conversions
    -- $customtypeconversions

    -- ** Dealing with bad data
    -- $baddata

    -- * Encoding and decoding
    -- $encoding
      HasHeader(..)
    , decode
    , decodeByName
    , encode
    , encodeByName

    -- ** Encoding and decoding options
    -- $options
    , DecodeOptions(..)
    , defaultDecodeOptions
    , decodeWith
    , decodeByNameWith
    , EncodeOptions(..)
    , defaultEncodeOptions
    , encodeWith
    , encodeByNameWith

    -- * Core CSV types
    , Csv
    , Record
    , Field
    , Header
    , Name
    , NamedRecord

    -- * Type conversion
    -- $typeconversion

    -- ** Index-based record conversion
    -- $indexbased
    , FromRecord(..)
    , Parser
    , runParser
    , index
    , (.!)
    , unsafeIndex
    , ToRecord(..)
    , record
    , Only(..)

    -- ** Name-based record conversion
    -- $namebased
    , FromNamedRecord(..)
    , lookup
    , (.:)
    , ToNamedRecord(..)
    , namedRecord
    , namedField
    , (.=)

    -- ** Field conversion
    -- $fieldconversion
    , FromField(..)
    , ToField(..)
    ) where

import Prelude hiding (lookup)

import Data.Csv.Conversion
import Data.Csv.Encoding
import Data.Csv.Types

-- $example
--
-- A short encoding usage example:
--
-- > >>> encode [("John" :: Text, 27), ("Jane", 28)]
-- > Chunk "John,27\r\nJane,28\r\n" Empty
--
-- Since string literals are overloaded we have to supply a type
-- signature as the compiler couldn't deduce which string type (i.e.
-- 'String' or 'Text') we want to use. In most cases type inference
-- will infer the type from the context and you can omit type
-- signatures.
--
-- A short decoding usage example:
--
-- > >>> decode NoHeader "John,27\r\nJane,28\r\n" :: Either String (Vector (Text, Int))
-- > Right (fromList [("John",27),("Jane",28)])
--
-- We pass 'NoHeader' as the first argument to indicate that the CSV
-- input data isn't preceded by a header.
--
-- In practice, the return type of 'decode' rarely needs to be given,
-- as it can often be inferred from the context.

-- $generic-processing
--
-- Sometimes you might want to work with a CSV file which contents is
-- unknown to you. For example, you might want remove the second
-- column of a file without knowing anything about its content. To
-- parse a CSV file to a generic representation, just convert each
-- record to a @'Vector' 'ByteString'@ value, like so:
--
-- > decode NoHeader "John,27\r\nJane,28\r\n" :: Either String (Vector (Vector ByteString))
-- > Right (fromList [fromList ["John","27"],fromList ["Jane","28"]])
--
-- As the example output above shows, all the fields are returned as
-- uninterpreted 'ByteString' values.

-- $customtypeconversions
--
-- Most of the time the existing 'FromField' and 'ToField' instances
-- do what you want. However, if you need to parse a different format
-- (e.g. hex) but use a type (e.g. 'Int') for which there's already a
-- 'FromField' instance, you need to use a @newtype@. Example:
--
-- > newtype Hex = Hex Int
-- >
-- > parseHex :: ByteString -> Parser Int
-- > parseHex = ...
-- >
-- > instance FromField Hex where
-- >     parseField s = Hex <$> parseHex s
--
-- Other than giving an explicit type signature, you can pattern match
-- on the @newtype@ constructor to indicate which type conversion you
-- want to have the library use:
--
-- > case decode NoHeader "0xff,0xaa\r\n0x11,0x22\r\n" of
-- >     Left err -> putStrLn err
-- >     Right v  -> forM_ v $ \ (Hex val1, Hex val2) ->
-- >         print (val1, val2)
--
-- If a field might be in one several different formats, you can use a
-- newtype to normalize the result:
--
-- > newtype HexOrDecimal = HexOrDecimal Int
-- >
-- > instance FromField DefaultToZero where
-- >     parseField s = case runParser (parseField s :: Parser Hex) of
-- >         Left err -> HexOrDecimal <$> parseField s  -- Uses Int instance
-- >         Right n  -> pure $ HexOrDecimal n
--
-- You can use the unit type, @()@, to ignore a column. The
-- 'parseField' method for @()@ doesn't look at the 'Field' and thus
-- always decodes successfully. Note that it lacks a corresponding
-- 'ToField' instance. Example:
--
-- > case decode NoHeader "foo,1\r\nbar,22" of
-- >     Left  err -> putStrLn err
-- >     Right v   -> forM_ v $ \ ((), i) -> print (i :: Int)

-- $baddata
--
-- If your input might contain invalid fields, you can write a custom
-- 'FromField' instance to deal with them. Example:
--
-- > newtype DefaultToZero = DefaultToZero Int
-- >
-- > instance FromField DefaultToZero where
-- >     parseField s = case runParser (parseField s) of
-- >         Left err -> pure $ DefaultToZero 0
-- >         Right n  -> pure $ DefaultToZero n

-- $encoding
--
-- Encoding and decoding is a two step process. To encode a value, it
-- is first converted to a generic representation, using either
-- 'ToRecord' or 'ToNamedRecord'. The generic representation is then
{'fraction_non_alphanumeric': 0.13188220230473752, 'fraction_numerical': 0.008568895892839554, 'mean_word_length': 2.5050051777701072, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 88, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1111480', 'n_tokens_mistral': 3167, 'n_tokens_neox': 3018, 'n_words': 1161}