<commit_before>{-| The API part of @feed-gipeda@. The console client is just a thin wrapper
    around this.
-}

module FeedGipeda
  ( Endpoint (..)
  , feedGipeda
  , module FeedGipeda.Types
  ) where


import           Control.Arrow                    (second)
import           Control.Concurrent               (forkIO)
import           Control.Concurrent.Chan          (Chan, newChan, readChan,
                                                   writeChan)
import qualified Control.Distributed.Backend.P2P  as P2P
import           Control.Distributed.Process      (Process, RemoteTable, liftIO,
                                                   say, spawnLocal)
import           Control.Distributed.Process.Node (initRemoteTable, runProcess)
import           Control.Logging                  as Logging
import           Control.Monad                    (forever, void, when)
import           Data.List                        (elemIndex)
import           Data.Maybe                       (fromMaybe, isJust)
import           Data.Set                         (Set)
import           Data.Time                        (NominalDiffTime)
import qualified FeedGipeda.Config                as Config
import qualified FeedGipeda.Gipeda                as Gipeda
import           FeedGipeda.GitShell              (SHA)
import qualified FeedGipeda.Master                as Master
import qualified FeedGipeda.Master.CommitQueue    as CommitQueue
import qualified FeedGipeda.Master.File           as Master.File
import           FeedGipeda.Prelude
import           FeedGipeda.Repo                  (Repo)
import qualified FeedGipeda.TaskScheduler         as TaskScheduler
import qualified FeedGipeda.THGenerated           as THGenerated
import           FeedGipeda.Types
import           Network.URI                      (parseURI)
import           System.Directory                 (getAppUserDataDirectory)
import           System.Exit                      (exitSuccess)
import           System.FilePath                  ((</>))


remoteTable :: RemoteTable
remoteTable =
  THGenerated.__remoteTable initRemoteTable


{-| The parameters correspond exactly to the command line parameters, to
    you should read the @--help@ message for more thorough documentation.

    @feedGipeda@ determines the appropriate mode of operation (e.g. watching or one-shot).
    It also works as master or slave node, depending on which endpoints are given.
    Lastly, @verbose@ will lead to more debug output.
-}
feedGipeda
  :: Paths
  -> Command
  -> Deployment
  -> ProcessRole
  -> Verbosity
  -> IO ()
feedGipeda paths cmd deployment role_ verbosity = do
  case verbosity of
    Verbose    -> Logging.setLogLevel Logging.LevelDebug
    NotVerbose -> Logging.setLogLevel Logging.LevelWarn

  case cmd of
    Check ->
      -- Just perform a syntax check on the given configFile
      Config.checkFile (configFile paths) >>= maybe exitSuccess error
    Build mode timeout -> do
      case slaveEndpoint role_ of
        Just (Endpoint host port) -> do
          let
            run = if isBoth role_ then void . forkIO else id
            toNodeId (Endpoint host port) =
              P2P.makeNodeId (host ++ ":" ++ show port)
            master = toNodeId (masterEndpoint role_)
          run (TaskScheduler.work "127.0.0.1" (show port) master remoteTable)
        _ -> return ()

      case (role_, masterEndpoint role_) of
        (Slave _ _, _) -> return ()
        (_, Endpoint host port) -> do
          queue <- CommitQueue.new
          P2P.bootstrap host (show port) [] remoteTable $ do
            let
              toTask :: (Repo, SHA) -> IO (TaskScheduler.Task String)
              toTask (repo, commit) = do
                script <- Gipeda.determineBenchmarkScript repo
                let closure = THGenerated.benchmarkClosure script repo commit timeout
                let finalize = Master.File.writeBenchmarkCSV repo commit . fromMaybe ""
                return (THGenerated.stringDict, closure, finalize)

            TaskScheduler.start (CommitQueue.dequeue queue >>= toTask)
            liftIO (Master.checkForNewCommits paths deployment mode queue)
<commit_msg>All localhost/127.0.0.1 stuff should be in FeedGipeda.Types<commit_after>{-| The API part of @feed-gipeda@. The console client is just a thin wrapper
    around this.
-}

module FeedGipeda
  ( Endpoint (..)
  , feedGipeda
  , module FeedGipeda.Types
  ) where


import           Control.Arrow                    (second)
import           Control.Concurrent               (forkIO)
import           Control.Concurrent.Chan          (Chan, newChan, readChan,
                                                   writeChan)
import qualified Control.Distributed.Backend.P2P  as P2P
import           Control.Distributed.Process      (Process, RemoteTable, liftIO,
                                                   say, spawnLocal)
import           Control.Distributed.Process.Node (initRemoteTable, runProcess)
import           Control.Logging                  as Logging
import           Control.Monad                    (forever, void, when)
import           Data.List                        (elemIndex)
import           Data.Maybe                       (fromMaybe, isJust)
import           Data.Set                         (Set)
import           Data.Time                        (NominalDiffTime)
import qualified FeedGipeda.Config                as Config
import qualified FeedGipeda.Gipeda                as Gipeda
import           FeedGipeda.GitShell              (SHA)
import qualified FeedGipeda.Master                as Master
import qualified FeedGipeda.Master.CommitQueue    as CommitQueue
import qualified FeedGipeda.Master.File           as Master.File
import           FeedGipeda.Prelude
import           FeedGipeda.Repo                  (Repo)
import qualified FeedGipeda.TaskScheduler         as TaskScheduler
import qualified FeedGipeda.THGenerated           as THGenerated
import           FeedGipeda.Types
import           Network.URI                      (parseURI)
import           System.Directory                 (getAppUserDataDirectory)
import           System.Exit                      (exitSuccess)
import           System.FilePath                  ((</>))


remoteTable :: RemoteTable
remoteTable =
  THGenerated.__remoteTable initRemoteTable


{-| The parameters correspond exactly to the command line parameters, to
    you should read the @--help@ message for more thorough documentation.

    @feedGipeda@ determines the appropriate mode of operation (e.g. watching or one-shot).
    It also works as master or slave node, depending on which endpoints are given.
    Lastly, @verbose@ will lead to more debug output.
-}
feedGipeda
  :: Paths
  -> Command
  -> Deployment
  -> ProcessRole
  -> Verbosity
  -> IO ()
feedGipeda paths cmd deployment role_ verbosity = do
  case verbosity of
    Verbose    -> Logging.setLogLevel Logging.LevelDebug
    NotVerbose -> Logging.setLogLevel Logging.LevelWarn

  case cmd of
    Check ->
      -- Just perform a syntax check on the given configFile
      Config.checkFile (configFile paths) >>= maybe exitSuccess error
    Build mode timeout -> do
      case slaveEndpoint role_ of
        Just (Endpoint shost sport) -> do
          let
            run = if isBoth role_ then void . forkIO else id
            Endpoint mhost mport = masterEndpoint role_
            master = P2P.makeNodeId (mhost ++ ":" ++ show mport)
          run (TaskScheduler.work shost (show sport) master remoteTable)
        _ -> return ()

      case (role_, masterEndpoint role_) of
        (Slave _ _, _) -> return ()
        (_, Endpoint host port) -> do
          queue <- CommitQueue.new
          P2P.bootstrap host (show port) [] remoteTable $ do
            let
              toTask :: (Repo, SHA) -> IO (TaskScheduler.Task String)
              toTask (repo, commit) = do
                script <- Gipeda.determineBenchmarkScript repo
                let closure = THGenerated.benchmarkClosure script repo commit timeout
                let finalize = Master.File.writeBenchmarkCSV repo commit . fromMaybe ""
                return (THGenerated.stringDict, closure, finalize)

            TaskScheduler.start (CommitQueue.dequeue queue >>= toTask)
            liftIO (Master.checkForNewCommits paths deployment mode queue)

{'fraction_non_alphanumeric': 0.06429171164687537, 'fraction_numerical': 0.0023989444644356484, 'mean_word_length': 1.8159405606214116, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 41, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1154845', 'n_tokens_mistral': 2138, 'n_tokens_neox': 2042, 'n_words': 668}