<commit_before>module TNUtils where

import qualified Data.ByteString.Lazy as L
import qualified Data.Binary as B
import System.Time
import Control.Concurrent
import Debug.Trace
import Data.List
import Control.Monad

cond :: [(Bool, a)] -> a -> a
cond [] x = x
cond ((True, x):_) _ = x
cond ((False, _):conds) x = cond conds x

nonempty = not . null

idInt :: Int -> Int
idInt = id

spliceFirst spl (s:ss) = (spl++s):ss

oneTrailingSlash "/" = "/"
oneTrailingSlash "" = ""
oneTrailingSlash s = case last s of
                      '/' -> s
                      _ -> s++"/"

inChunksOf :: Int -> [a] -> [[a]]
inChunksOf _ [] = []
inChunksOf n xs = let (hds, tls) = splitAt n xs 
                  in hds : inChunksOf n tls

orJust (Just x) _ = x
orJust _ y = y 


saveBinary :: B.Binary w => FilePath-> w -> IO ()
saveBinary fp w = L.writeFile fp {-. compress-} . B.encode $ w --writeFile fp . show

appendBinary :: B.Binary w => FilePath-> w -> IO ()
appendBinary fp w = L.appendFile fp {-. compress-} . B.encode $ w --writeFile fp . show

loadBinary :: B.Binary w =>FilePath-> IO w
loadBinary fp = return . B.decode {-. decompress -}=<< L.readFile fp --readFile fp >>= return . read

lookupMany :: Eq a => a -> [(a,b)] -> [b]
lookupMany x = map snd . filter ((==x) . fst)

--http://www.serpentine.com/blog/2009/01/11/fun-with-haskell-view-patterns/
dropPrefix :: Eq a => [a] -> [a] -> ([a],[a])
dropPrefix left@(x:xs) right@(y:ys)
    | x == y    = dropPrefix xs ys
dropPrefix left right = (left,right)



for = flip map

safeHead [] = Nothing
safeHead (x:_) = Just x

safeLast [] = Nothing
safeLast xs = Just $ last xs
<commit_msg>safeRead<commit_after>module TNUtils where

import qualified Data.ByteString.Lazy as L
import qualified Data.Binary as B
import System.Time
import Control.Concurrent
import Debug.Trace
import Data.List
import Control.Monad

cond :: [(Bool, a)] -> a -> a
cond [] x = x
cond ((True, x):_) _ = x
cond ((False, _):conds) x = cond conds x

nonempty = not . null

idInt :: Int -> Int
idInt = id

spliceFirst spl (s:ss) = (spl++s):ss

oneTrailingSlash "/" = "/"
oneTrailingSlash "" = ""
oneTrailingSlash s = case last s of
                      '/' -> s
                      _ -> s++"/"

noInitSlash ('/':s) = s
noInitSlash s = s

x ./ y = oneTrailingSlash x ++ noInitSlash y

inChunksOf :: Int -> [a] -> [[a]]
inChunksOf _ [] = []
inChunksOf n xs = let (hds, tls) = splitAt n xs 
                  in hds : inChunksOf n tls

orJust (Just x) _ = x
orJust _ y = y 


saveBinary :: B.Binary w => FilePath-> w -> IO ()
saveBinary fp w = L.writeFile fp {-. compress-} . B.encode $ w --writeFile fp . show

appendBinary :: B.Binary w => FilePath-> w -> IO ()
appendBinary fp w = L.appendFile fp {-. compress-} . B.encode $ w --writeFile fp . show

loadBinary :: B.Binary w =>FilePath-> IO w
loadBinary fp = return . B.decode {-. decompress -}=<< L.readFile fp --readFile fp >>= return . read

lookupMany :: Eq a => a -> [(a,b)] -> [b]
lookupMany x = map snd . filter ((==x) . fst)

--http://www.serpentine.com/blog/2009/01/11/fun-with-haskell-view-patterns/
dropPrefix :: Eq a => [a] -> [a] -> ([a],[a])
dropPrefix left@(x:xs) right@(y:ys)
    | x == y    = dropPrefix xs ys
dropPrefix left right = (left,right)



for = flip map

safeHead [] = Nothing
safeHead (x:_) = Just x

safeLast [] = Nothing
safeLast xs = Just $ last xs

safeRead :: Read a => String -> Maybe a
safeRead x = case readsPrec 5 x of 
               [] -> Nothing
               (x,_):_ -> Just x
               


{'fraction_non_alphanumeric': 0.1619156214367161, 'fraction_numerical': 0.004846066134549601, 'mean_word_length': 2.56243654822335, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3331883', 'n_tokens_mistral': 1282, 'n_tokens_neox': 1232, 'n_words': 407}