<commit_before>{-# LANGUAGE DeriveGeneric #-}

import qualified Network as N
import Network.HTTP.Client

import System.IO
import System.Environment

import Control.Concurrent
import Control.Monad

import qualified Data.ByteString.Lazy.Char8 as BL
import Data.Aeson (FromJSON, ToJSON, decode, encode)
import Data.String

import GHC.Generics (Generic)

import qualified ArgumentsParser as AP
import qualified Data.Text as DT
import qualified GetWordsFile as GWF
import qualified PorterStemmer as PS
import qualified StatisticsSlave as Stat
import qualified KeyWordOccurrence as KWO

main :: IO ()
main = do
    print "Connecting to Master @172.16.58.3:4444"
    handle <- N.connectTo "172.16.58.3" (N.PortNumber 4444)
    putStrLn "Connected"
    socket <- N.listenOn (N.PortNumber 5000)
    incomingOrder <- newChan
    (handle, host, port) <- N.accept socket
    hSetBuffering handle LineBuffering
    forkIO $ forever $ receiveOrder handle incomingOrder
    forkIO $ forever $ executeOrder incomingOrder
    forever $ do
        putStrLn "Slave is active..."
        threadDelay 5000000

-- |Receives and broadcasts next order on incomingOrder channel
receiveOrder :: Handle -> Chan [[String]] -> IO ()
receiveOrder handle incomingOrder = do
    code <- hGetLine handle
    let order = read code :: [[String]]
    print $ "Order received : " ++ show order
    writeChan incomingOrder order
    -- receiveOrder handle incomingOrder

-- | Reads and executes next order
executeOrder :: Chan [[String]] -> IO ()
executeOrder incomingOrder = do
    order <- readChan incomingOrder
    let parsedArguments = map AP.parseArguments order
    let urls = map extractURLs parsedArguments
    descriptions <- getAllContent urls
    let work = process parsedArguments descriptions
    handle <- N.connectTo "172.16.58.3" (N.PortNumber 4446)
    hSetBuffering handle LineBuffering
    hPutStrLn handle work
    hClose handle

extractURLs :: [(String, Int, Int, String)] -> [String]
extractURLs [] = []
extractURLs (x:xs) = (getURL x) : (extractURLs xs)

getURL :: (String, Int, Int, String) -> String
getURL (_,_,_,u) = u

get :: String -> IO String
get url = do
    manager <- newManager defaultManagerSettings
    request <- parseRequest url
    response <- httpLbs request manager
    return $ BL.unpack $ responseBody response
<commit_msg>Forever-ing functions<commit_after>{-# LANGUAGE DeriveGeneric #-}

import qualified Network as N
import Network.HTTP.Client

import System.IO
import System.Environment

import Control.Concurrent
import Control.Monad

import qualified Data.ByteString.Lazy.Char8 as BL
import Data.Aeson (FromJSON, ToJSON, decode, encode)
import Data.String

import GHC.Generics (Generic)

import qualified ArgumentsParser as AP
import qualified Data.Text as DT
import qualified GetWordsFile as GWF
import qualified PorterStemmer as PS
import qualified StatisticsSlave as Stat
import qualified KeyWordOccurrence as KWO

main :: IO ()
main = do
    print "Connecting to Master @172.16.58.3:4444"
    handle <- N.connectTo "172.16.58.3" (N.PortNumber 4444)
    putStrLn "Connected"
    socket <- N.listenOn (N.PortNumber 5000)
    incomingOrder <- newChan
    outgoingOrder <- newChan
    (handle, host, port) <- N.accept socket
    hSetBuffering handle LineBuffering
    forkIO $ receiveOrder handle incomingOrder
    forkIO $ executeOrder incomingOrder outgoingOrder
    forkIO $ prepareToSendWork outgoingOrder
    forever $ do
        putStrLn "Slave is active..."
        threadDelay 5000000

-- |Receives and broadcasts next order on incomingOrder channel
receiveOrder :: Handle -> Chan [[String]] -> IO ()
receiveOrder handle incomingOrder = do
    code <- hGetLine handle
    let order = read code :: [[String]]
    print $ "Order received : " ++ show order
    writeChan incomingOrder order
    receiveOrder handle incomingOrder
    -- receiveOrder handle incomingOrder

-- | Reads and executes next order
executeOrder    :: Chan [[String]] 
                -> Chan String 
                -> IO ()
executeOrder incomingOrder outgoingOrder = do
    order <- readChan incomingOrder
    let parsedArguments = map AP.parseArguments order
    let urls = map extractURLs parsedArguments
    descriptions <- getAllContent urls
    let work = process parsedArguments descriptions
    writeChan outgoingOrder work
    executeOrder incomingOrder outgoingOrder

prepareToSendWork :: Chan String -> IO ()
prepareToSendWork outgoingOrder = do
    handle <- N.connectTo "172.16.58.3" (N.PortNumber 4446)
    hSetBuffering handle LineBuffering
    forever $ sendWork outgoingOrder handle
    
sendWork :: Chan String -> Handle -> IO ()
sendWork outgoingOrder handle = do
    work <- readChan outgoingOrder
    hPutStrLn handle work
    
extractURLs :: [(String, Int, Int, String)] -> [String]
extractURLs [] = []
extractURLs (x:xs) = (getURL x) : (extractURLs xs)

getURL :: (String, Int, Int, String) -> String
getURL (_,_,_,u) = u

get :: String -> IO String
get url = do
    manager <- newManager defaultManagerSettings
    request <- parseRequest url
    response <- httpLbs request manager
    return $ BL.unpack $ responseBody response

{'fraction_non_alphanumeric': 0.08258841818888457, 'fraction_numerical': 0.018655266226195105, 'mean_word_length': 3.6244384546271338, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 6, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3657232', 'n_tokens_mistral': 1546, 'n_tokens_neox': 1421, 'n_words': 510}