<commit_before>{-# LANGUAGE OverloadedStrings, RecordWildCards #-}

module Main (
      main
    ) where

import Control.Concurrent     (forkIO)
import Control.Monad.STM      (atomically)
import Control.Monad.IO.Class (liftIO)
import Data.Conduit
import Data.Conduit.Binary    (sourceHandle, sinkHandle)
import Data.Conduit.TMChan
import Data.Maybe             (fromJust)
import Network.AMQP.Conduit
import System.IO              (stdin, stdout)
import Bark.Conduit
import Bark.Options

import qualified Data.ByteString as BS

source :: TBMChan BS.ByteString -> IO ()
source chan =
     runResourceT
         $  sourceHandle stdin
         $$ sinkTBMChan chan

conduits :: MonadResource m
         => Options
         -> Conduit BS.ByteString m BS.ByteString
conduits Options{..} =
    (conduitSplit $ fromString optDelimiter) -- =$= (conduitHandle stdout)

sink :: TBMChan BS.ByteString -> Options -> IO ()
sink chan opts =
    runResourceT
        $  sourceTBMChan chan
        $= conduits opts
--        $$ sinkAMQP uri exchange queue
        $$ sinkHandle stdout
  where
    uri      = fromJust $ parseURI "amqp://guest:guest@127.0.0.1/"
    exchange = newExchange { exchangeName = "test", exchangeType = "direct" }
    queue    = newQueue { queueName = "test" }

main :: IO ()
main = do
    opts@Options{..} <- parseOptions
    putStrLn $ show opts
    putStrLn . show $ fromString optDelimiter

    chan <- atomically $ newTBMChan optBound
    _    <- forkIO $ source chan
    sink chan opts<commit_msg>Removing debug output<commit_after>{-# LANGUAGE OverloadedStrings, RecordWildCards #-}

module Main (
      main
    ) where

import Control.Concurrent     (forkIO)
import Control.Monad.STM      (atomically)
import Control.Monad.IO.Class (liftIO)
import Data.Conduit
import Data.Conduit.Binary    (sourceHandle, sinkHandle)
import Data.Conduit.TMChan
import Data.Maybe             (fromJust)
import Network.AMQP.Conduit
import System.IO              (stdin, stdout)
import Bark.Conduit
import Bark.Options
import Data.ByteString.Char8 (pack)
import qualified Data.ByteString as BS

source :: TBMChan BS.ByteString -> IO ()
source chan =
     runResourceT
         $  sourceHandle stdin
         $$ sinkTBMChan chan

conduits :: MonadResource m
         => Options
         -> Conduit BS.ByteString m BS.ByteString
conduits Options{..} =
    (conduitSplit $ fromString optDelimiter) -- =$= (conduitHandle stdout)

sink :: TBMChan BS.ByteString -> Options -> IO ()
sink chan opts =
    runResourceT
        $  sourceTBMChan chan
        $= conduits opts
--        $$ sinkAMQP uri exchange queue
        $$ sinkHandle stdout
  where
    uri      = fromJust $ parseURI "amqp://guest:guest@127.0.0.1/"
    exchange = newExchange { exchangeName = "test", exchangeType = "direct" }
    queue    = newQueue { queueName = "test" }

main :: IO ()
main = do
    opts@Options{..} <- parseOptions
    putStrLn $ show opts

    chan <- atomically $ newTBMChan optBound
    _    <- forkIO $ source chan
    sink chan opts
{'fraction_non_alphanumeric': 0.09817578772802653, 'fraction_numerical': 0.004311774461028192, 'mean_word_length': 2.732673267326733, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '445208', 'n_tokens_mistral': 985, 'n_tokens_neox': 935, 'n_words': 252}