<commit_before>
-- | Thread pool implementation.
module Development.Shake.Pool(Pool, addPool, blockPool, runPool) where

import Control.Concurrent
import Control.Exception hiding (blocked)
import Development.Shake.Locks
import qualified Data.HashSet as Set


---------------------------------------------------------------------
-- SUPER QUEUE

-- FIXME: The super queue should use randomness for the normal priority pile

data SuperQueue a = SuperQueue [a] [a]

newSuperQueue :: SuperQueue a
newSuperQueue = SuperQueue [] []

enqueuePriority :: a -> SuperQueue a -> SuperQueue a
enqueuePriority x (SuperQueue p n) = SuperQueue (x:p) n

enqueue :: a -> SuperQueue a -> SuperQueue a
enqueue x (SuperQueue p n) = SuperQueue p (x:n)

dequeue :: SuperQueue a -> Maybe (a, SuperQueue a)
dequeue (SuperQueue (p:ps) ns) = Just (p, SuperQueue ps ns)
dequeue (SuperQueue [] (n:ns)) = Just (n, SuperQueue [] ns)
dequeue (SuperQueue [] []) = Nothing


---------------------------------------------------------------------
-- THREAD POOL

{-
Must keep a list of active threads, so can raise exceptions in a timely manner
Must spawn a fresh thread to do blockPool
If any worker throws an exception, must signal to all the other workers
-}

data Pool = Pool Int (Var (Maybe S)) (Barrier (Maybe SomeException))

data S = S
    {threads :: Set.HashSet ThreadId
    ,working :: Int -- threads which are actively working
    ,blocked :: Int -- threads which are blocked
    ,todo :: SuperQueue (IO ())
    }


emptyS :: S
emptyS = S Set.empty 0 0 newSuperQueue<commit_msg>Rename from Super Queue to Random Queue<commit_after>
-- | Thread pool implementation.
module Development.Shake.Pool(Pool, addPool, blockPool, runPool) where

import Control.Concurrent
import Control.Exception hiding (blocked)
import Development.Shake.Locks
import qualified Data.HashSet as Set


---------------------------------------------------------------------
-- RANDOM QUEUE

-- FIXME: The queue should use randomness for the normal priority pile

data RandomQueue a = RandomQueue [a] [a]

newRandomQueue :: RandomQueue a
newRandomQueue = RandomQueue [] []

enqueuePriority :: a -> RandomQueue a -> RandomQueue a
enqueuePriority x (RandomQueue p n) = RandomQueue (x:p) n

enqueue :: a -> RandomQueue a -> RandomQueue a
enqueue x (RandomQueue p n) = RandomQueue p (x:n)

dequeue :: RandomQueue a -> Maybe (a, RandomQueue a)
dequeue (RandomQueue (p:ps) ns) = Just (p, RandomQueue ps ns)
dequeue (RandomQueue [] (n:ns)) = Just (n, RandomQueue [] ns)
dequeue (RandomQueue [] []) = Nothing


---------------------------------------------------------------------
-- THREAD POOL

{-
Must keep a list of active threads, so can raise exceptions in a timely manner
Must spawn a fresh thread to do blockPool
If any worker throws an exception, must signal to all the other workers
-}

data Pool = Pool Int (Var (Maybe S)) (Barrier (Maybe SomeException))

data S = S
    {threads :: Set.HashSet ThreadId
    ,working :: Int -- threads which are actively working
    ,blocked :: Int -- threads which are blocked
    ,todo :: RandomQueue (IO ())
    }


emptyS :: S
emptyS = S Set.empty 0 0 newRandomQueue
{'fraction_non_alphanumeric': 0.17766497461928935, 'fraction_numerical': 0.0012690355329949238, 'mean_word_length': 2.683411214953271, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5867686', 'n_tokens_mistral': 873, 'n_tokens_neox': 859, 'n_words': 364}