<commit_before>{-# OPTIONS_GHC  -XNoImplicitPrelude -funbox-strict-fields #-}
{-# LANGUAGE BangPatterns #-}
-----------------------------------------------------------------------------
-- |
-- Module      :  GHC.IO.Encoding.UTF16
-- Copyright   :  (c) The University of Glasgow, 2009
-- License     :  see libraries/base/LICENSE
-- 
-- Maintainer  :  <EMAIL>
-- Stability   :  internal
-- Portability :  non-portable
--
-- UTF-16 Codecs for the IO library
--
-- Portions Copyright   : (c) <NAME> 2008-2009,
--                        (c) <NAME> 2009,
--                        (c) <NAME> 2009
--
-----------------------------------------------------------------------------

module GHC.IO.Encoding.UTF16 (
  utf16,
  utf16_decode,
  utf16_encode,

  utf16be,
  utf16be_decode,
  utf16be_encode,

  utf16le,
  utf16le_decode,
  utf16le_encode,
  ) where

import GHC.Base
import GHC.Real
import GHC.Num
-- import GHC.IO
import GHC.IO.Exception
import GHC.IO.Buffer
import GHC.IO.Encoding.Types
import GHC.Word
import Data.Bits
import Data.Maybe
import GHC.IORef

#if DEBUG
import System.Posix.Internals
import Foreign.C
import GHC.Show

puts :: String -> IO ()
puts s = do withCStringLen (s++"\n") $ \(p,len) -> 
                c_write 1 p (fromIntegral len)
            return ()
#endif

-- -----------------------------------------------------------------------------
-- The UTF-16 codec: either UTF16BE or UTF16LE with a BOM

utf16  :: TextEncoding
utf16 = TextEncoding { mkTextDecoder = utf16_DF,
 	               mkTextEncoder = utf16_EF }

utf16_DF :: IO (TextDecoder (Maybe DecodeBuffer))
utf16_DF = do
  seen_bom <- newIORef Nothing
  return (BufferCodec {
             encode   = utf16_decode seen_bom,
             close    = return (),
             getState = readIORef seen_bom,
             setState = writeIORef seen_bom
          })<commit_msg>Add a missing castPtr (only shows up in -DDEBUG)<commit_after>{-# OPTIONS_GHC  -XNoImplicitPrelude -funbox-strict-fields #-}
{-# LANGUAGE BangPatterns #-}
-----------------------------------------------------------------------------
-- |
-- Module      :  GHC.IO.Encoding.UTF16
-- Copyright   :  (c) The University of Glasgow, 2009
-- License     :  see libraries/base/LICENSE
-- 
-- Maintainer  :  <EMAIL>
-- Stability   :  internal
-- Portability :  non-portable
--
-- UTF-16 Codecs for the IO library
--
-- Portions Copyright   : (c) <NAME> 2008-2009,
--                        (c) <NAME> 2009,
--                        (c) <NAME> 2009
--
-----------------------------------------------------------------------------

module GHC.IO.Encoding.UTF16 (
  utf16,
  utf16_decode,
  utf16_encode,

  utf16be,
  utf16be_decode,
  utf16be_encode,

  utf16le,
  utf16le_decode,
  utf16le_encode,
  ) where

import GHC.Base
import GHC.Real
import GHC.Num
-- import GHC.IO
import GHC.IO.Exception
import GHC.IO.Buffer
import GHC.IO.Encoding.Types
import GHC.Word
import Data.Bits
import Data.Maybe
import GHC.IORef

#if DEBUG
import System.Posix.Internals
import Foreign.C
import GHC.Show
import GHC.Ptr

puts :: String -> IO ()
puts s = do withCStringLen (s++"\n") $ \(p,len) -> 
                c_write 1 (castPtr p) (fromIntegral len)
            return ()
#endif

-- -----------------------------------------------------------------------------
-- The UTF-16 codec: either UTF16BE or UTF16LE with a BOM

utf16  :: TextEncoding
utf16 = TextEncoding { mkTextDecoder = utf16_DF,
 	               mkTextEncoder = utf16_EF }

utf16_DF :: IO (TextDecoder (Maybe DecodeBuffer))
utf16_DF = do
  seen_bom <- newIORef Nothing
  return (BufferCodec {
             encode   = utf16_decode seen_bom,
             close    = return (),
             getState = readIORef seen_bom,
             setState = writeIORef seen_bom
          })
{'fraction_non_alphanumeric': 0.22033898305084745, 'fraction_numerical': 0.034427966101694914, 'mean_word_length': 1.5834473324213407, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5987325', 'n_tokens_mistral': 1280, 'n_tokens_neox': 1158, 'n_words': 291}