<commit_before>{-# LANGUAGE
      TypeSynonymInstances
    , FlexibleInstances
  #-}


module OskarBinReader where

import OskarBinReaderFFI
import Foreign
import Foreign.C
import Foreign.Storable.Complex ()
import Text.Printf

data TaskData = TaskData {
    tdBaselines :: !Int
  , tdTimes     :: !Int
  , tdChannels  :: !Int
  , tdPoints    :: !Int
  , tdWstep     :: !CDouble
  , tdVisibilies :: !(ForeignPtr CxDouble)
  , tdUVWs :: !(ForeignPtr CDouble)
  , tdMap  :: !(ForeignPtr BlWMap)
}

readOskarData :: String -> IO TaskData
readOskarData fname = withCString fname doRead
  where
    fi = fromIntegral
    throwErr = throwIf_ (/= 0) (\n -> printf "While trying to read binary file %s : %d" fname $ fi n)
    doRead namep = alloca $ \vptr -> do
      throwErr $ mkFromFile vptr namep
      nbls <- numBaselines vptr
      ntms <- numTimes    vptr
      nchs <- numChannels vptr
      -- it is in fact a product of previous three
      npts <- numPoints vptr
      let
        n = fi npts
        nb = fi nbls
        visSize = n * sizeOf (undefined :: CxDouble) * 4
        uvwSize = n * sizeOf (undefined :: CDouble) * 3
        mapSize = nb * sizeOf (undefined :: BlWMap)
      visFptr <- mallocForeignPtrBytes visSize
      uvwFptr <- mallocForeignPtrBytes uvwSize
      mapFptr <- mallocForeignPtrBytes mapSize
      alloca $ \mptr ->
        allocaBytes (nb * sizeOf (undefined :: WMaxMin)) $ \mmptr ->
          withForeignPtr visFptr $ \visptr ->
            withForeignPtr uvwFptr $ \uwvptr ->
              withForeignPtr mapFptr $ \mapptr -> do
                throwErr $ readAndReshuffle vptr visptr uwvptr mptr mmptr mapptr
                wstp <- wstep mptr
                return $ TaskData nb (fi ntms) (fi nchs) n wstp visFptr uvwFptr mapFptr<commit_msg>Switch to manual memory management for Oskar binary reader. This also allows to serialize 'TaskData'.<commit_after>{-# LANGUAGE
      TypeSynonymInstances
    , FlexibleInstances
  #-}


module OskarBinReader where

import OskarBinReaderFFI
import Foreign
import Foreign.C
import Foreign.Storable.Complex ()
import Text.Printf

data TaskData = TaskData {
    tdBaselines :: !Int
  , tdTimes     :: !Int
  , tdChannels  :: !Int
  , tdPoints    :: !Int
  , tdWstep     :: !CDouble
  , tdVisibilies :: !(Ptr CxDouble)
  , tdUVWs :: !(Ptr CDouble)
  , tdMap  :: !(Ptr BlWMap)
}

finalizeTaskData :: TaskData -> IO ()
finalizeTaskData td = do
  free $ tdMap td
  free $ tdUVWs td
  free $ tdVisibilies td

readOskarData :: String -> IO TaskData
readOskarData fname = withCString fname doRead
  where
    fi = fromIntegral
    throwErr = throwIf_ (/= 0) (\n -> printf "While trying to read binary file %s : %d" fname $ fi n)
    doRead namep = alloca $ \vptr -> do
      throwErr $ mkFromFile vptr namep
      nbls <- numBaselines vptr
      ntms <- numTimes    vptr
      nchs <- numChannels vptr
      -- it is in fact a product of previous three
      npts <- numPoints vptr
      let
        n = fi npts
        nb = fi nbls
        visSize = n * sizeOf (undefined :: CxDouble) * 4
        uvwSize = n * sizeOf (undefined :: CDouble) * 3
        mapSize = nb * sizeOf (undefined :: BlWMap)
      visptr <- mallocBytes visSize
      uvwptr <- mallocBytes uvwSize
      mapptr <- mallocBytes mapSize
      alloca $ \mptr ->
        allocaBytes (nb * sizeOf (undefined :: WMaxMin)) $ \mmptr -> do
          throwErr $ readAndReshuffle vptr visptr uvwptr mptr mmptr mapptr
          wstp <- wstep mptr
          return $ TaskData nb (fi ntms) (fi nchs) n wstp visptr uvwptr mapptr
{'fraction_non_alphanumeric': 0.08893892485223755, 'fraction_numerical': 0.0016887137630171687, 'mean_word_length': 2.4107485604606524, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1207745', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1169, 'n_words': 361}