<commit_before>import qualified Data.ByteString.Lazy as L
import qualified Data.ByteString as B
import qualified Data.Map as Map

import System.IO
import Data.Word
import Data.Maybe (fromJust)
import Data.Array
import Control.Monad (liftM)

import Contract.Types
import Contract.Constants
import Contract.Protocol (decodeFileHeader, decodeTick)
import Utils.Array (sampleAtInterval)

-- | Converts a list of ByteStrings into a list of Ticks, assuming that interval is 1 sec.
getSecondIntervalData :: [L.ByteString] -> Word32 -> [Tick]
getSecondIntervalData bs points = undefined

decodeTicks :: [L.ByteString] -> [Tick]
decodeTicks = undefined

-- | TODO: handle Nothing which signifies an exception
readHeader ::  Handle -> IO Header
readHeader fh = liftM (fromJust . decodeFileHeader) $ L.hGet fh bytesInHeader
    where bytesInHeader = 5 * wordSize

{- | Samples byte strings from a given file handle
     h          - the file handle
     maxPoint   - the point after which to stop reading (usually the end of the file)
     interval   - interval at which to read points -}
sampleByteStrings :: Handle -> Int -> Int -> Int -> IO (Array Int L.ByteString)
sampleByteStrings h maxPoint interval pointSize = 
    liftM (listArray (0, points - 1)) $ sample points []
    where points = maxPoint `div` interval
          skipSize = pointSize * (interval - 1)
          sample remainingPoints xs = do 
              x <- L.hGet h pointSize
              if remainingPoints >= 0 then do 
                   hSeek h RelativeSeek $ fromIntegral skipSize
                   sample (remainingPoints - 1) (x:xs)
              else return $ reverse xs

getFullSymbolDataPath :: FilePath -> Symbol -> FilePath
getFullSymbolDataPath dataDir symbolName = dataDir ++ "/" ++ symbolName ++ ".bin"

readTickDataAtInterval :: FilePath -> TimeInterval -> IO ((Array Int L.ByteString), Header)
readTickDataAtInterval fullPath interval = do
    h <- openBinaryFile fullPath ReadMode
    hdr <- readHeader h
    sample <- sampleByteStrings h (fromIntegral $ points hdr) (fromIntegral interval) tickSize 
    hClose h
    return (sample, hdr)

getHistoricalTickData :: FilePath -> IO HistoricalTickData
getHistoricalTickData fullPath = do 
    (byMinute, hdr) <- readTickDataAtInterval fullPath secondsPerMinute 
    let byHour = sampleAtInterval (fromIntegral minutesPerHour) byMinute
    let byDay  = sampleAtInterval (fromIntegral hoursPerDay) byHour 
    return $ HistoricalTickData 
                (symbol hdr) 
                (Map.fromList [ (secondsPerMinute, byMinute)
                              , (secondsPerHour, byHour)
                              , (secondsPerDay, byDay)
                              ])

-----------------------
-- ----  Tests  ---- --
-----------------------

debug = do

    tickData <- getHistoricalTickData $ getFullSymbolDataPath dataDir symbolName

    let hourly = (fromJust . Map.lookup secondsPerHour) $ tickDataByInterval tickData
    let daily = (fromJust . Map.lookup secondsPerDay) $ tickDataByInterval tickData

    putStrLn "Hourly Data:" 
    print $ (map decodeTick . elems) hourly
    putStrLn "Daily Data:" 
    print $ (map decodeTick . elems) daily
    
    putStrLn "\n--------------\nDone!"

    where dataDir = "/Users/thlorenz/dev/data/Pricetory"
          symbolName = "EURUSD"

-- Reader monad via data-ivar, write-once variables
-- http://hackage.haskell.org/packages/archive/data-ivar/0.30/doc/html/Data-IVar.html
<commit_msg>first go at getting world of tick data that checks out<commit_after>module Data.Sampler where

import qualified Data.ByteString.Lazy as L
import qualified Data.ByteString as B
import qualified Data.Map as Map

import System.IO
import Data.Word
import Data.Maybe (fromJust)
import Data.Array
import Control.Monad (liftM)

import Contract.Types
import Contract.Constants
import Contract.Protocol (decodeFileHeader, decodeTick)
import Utils.Array (sampleAtInterval)

-- | TODO: handle Nothing which signifies an exception
readHeader ::  Handle -> IO Header
readHeader fh = liftM (fromJust . decodeFileHeader) $ L.hGet fh bytesInHeader
    where bytesInHeader = 5 * wordSize

{- | Samples byte strings from a given file handle
     h          - the file handle
     maxPoint   - the point after which to stop reading (usually the end of the file)
     interval   - interval at which to read points -}
sampleByteStrings :: Handle -> Int -> Int -> Int -> IO (Array Int L.ByteString)
sampleByteStrings h maxPoint interval pointSize = 
    liftM (listArray (0, points - 1)) $ sample points []
    where points = maxPoint `div` interval
          skipSize = pointSize * (interval - 1)
          sample remainingPoints xs = do 
              x <- L.hGet h pointSize
              if remainingPoints >= 0 then do 
                   hSeek h RelativeSeek $ fromIntegral skipSize
                   sample (remainingPoints - 1) (x:xs)
              else return $ reverse xs

getFullSymbolDataPath :: FilePath -> Symbol -> FilePath
getFullSymbolDataPath dataDir symbolName = dataDir ++ "/" ++ symbolName ++ ".bin"

readTickDataAtInterval :: FilePath -> TimeInterval -> IO ((Array Int L.ByteString), Header)
readTickDataAtInterval fullPath interval = do
    h <- openBinaryFile fullPath ReadMode
    hdr <- readHeader h
    sample <- sampleByteStrings h (fromIntegral $ points hdr) (fromIntegral interval) tickSize 
    hClose h
    return (sample, hdr)

getHistoricalTickData :: FilePath -> IO HistoricalTickData
getHistoricalTickData fullPath = do 
    (byMinute, hdr) <- readTickDataAtInterval fullPath secondsPerMinute 
    let byHour = sampleAtInterval (fromIntegral minutesPerHour) byMinute
    let byDay  = sampleAtInterval (fromIntegral hoursPerDay) byHour 
    return $ HistoricalTickData 
                (symbol hdr) 
                (Map.fromList [ (secondsPerMinute, byMinute)
                              , (secondsPerHour, byHour)
                              , (secondsPerDay, byDay)
                              ])

getWorldOfTickData :: FilePath -> [Symbol] -> IO HistoricalTickDataMap
getWorldOfTickData dataDir = do
    tpls <- mapM (liftM toTuple . getHistoricalTickData . getFullSymbolDataPath dataDir)
    let mapped = liftM Map.fromList $ tpls
    return $ liftM HistoricalTickDataMap $ mapped
    where 
        toTuple htd = (tickDataSymbol htd, htd)
-----------------------
-- ----  Tests  ---- --
-----------------------

debug = do

    tickData <- getHistoricalTickData $ getFullSymbolDataPath dataDir symbolName

    let hourly = (fromJust . Map.lookup secondsPerHour) $ tickDataByInterval tickData
    let daily = (fromJust . Map.lookup secondsPerDay) $ tickDataByInterval tickData

    putStrLn "Hourly Data:" 
    print $ (map decodeTick . elems) hourly
    putStrLn "Daily Data:" 
    print $ (map decodeTick . elems) daily
    
    putStrLn "\n--------------\nDone!"

    where dataDir = "/Users/thlorenz/dev/data/Pricetory"
          symbolName = "EURUSD"

-- Reader monad via data-ivar, write-once variables
-- http://hackage.haskell.org/packages/archive/data-ivar/0.30/doc/html/Data-IVar.html

{'fraction_non_alphanumeric': 0.09705177034842714, 'fraction_numerical': 0.0029623360135421074, 'mean_word_length': 2.679294239750908, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3175792', 'n_tokens_mistral': 2051, 'n_tokens_neox': 1945, 'n_words': 632}