<commit_before>{-# LANGUAGE OverloadedStrings #-}

import Control.Concurrent
import Control.Concurrent.Async
import Control.Exception (Exception, throw)
import Control.Monad
import Data.List
import qualified Filesystem.Path.CurrentOS as FP
import GHC.Conc
import System.Directory
import System.FilePath.Posix
import System.FSNotify
import System.Path.NameManip
import System.Posix.Process

import Control.Concurrent.Chan.Len
import Pygmalion.Analysis.Extension
import Pygmalion.Analysis.Manager
import Pygmalion.Config
import Pygmalion.Core
import Pygmalion.Database.Manager
import Pygmalion.Log
import Pygmalion.RPC.Server

main :: IO ()
main = do
  cf <- getConfiguration
  initLogger (logLevel cf)
  nice 5
  ensureDB
  stopWatching <- newEmptyMVar
  port <- newEmptyMVar
  aChan <- newLenChan
  dbChan <- newLenChan
  dbQueryChan <- newLenChan
  logDebug "Launching database thread"
  dbThread <- asyncBound (runDatabaseManager dbChan dbQueryChan)
  --let maxThreads = numCapabilities
  let maxThreads = 4 :: Int
  threads <- forM [1..maxThreads] $ \i -> do
    logDebug $ "Launching analysis thread #" ++ (show i)
    asyncBound (runAnalysisManager aChan dbChan dbQueryChan)
  rpcThread <- async (runRPCServer cf port aChan dbQueryChan)
  watchThread <- async (doWatch aChan stopWatching)
  _ <- getLine
  cancel rpcThread
  --logDebug "Just terminated RPC thread."
  --_ <- getLine
  putMVar stopWatching ()
  wait watchThread
  --logDebug "Just terminated watch thread."
  --_ <- getLine
  forM_ threads $ \_ -> writeLenChan aChan ShutdownAnalysis  -- Signifies end of data.
  forM_ (zip threads [1..numCapabilities]) $ \(thread, i) -> do
    ensureNoException =<< waitCatch thread
    logDebug $ "Termination of thread #" ++ (show i)
  writeLenChan dbChan DBShutdown  -- Terminate the database thread.
  ensureNoException =<< waitCatch dbThread
  logDebug $ "Termination of database thread"

doWatch :: AnalysisChan -> MVar () -> IO ()
doWatch aChan stopWatching = do
  -- Restart every 10 minutes until a better fix is found. =(<commit_msg>Link the main pygd thread to all of the child threads. Closes #38<commit_after>{-# LANGUAGE OverloadedStrings #-}

import Control.Concurrent
import Control.Concurrent.Async
import Control.Exception (Exception, throw)
import Control.Monad
import Data.List
import qualified Filesystem.Path.CurrentOS as FP
import GHC.Conc
import System.Directory
import System.FilePath.Posix
import System.FSNotify
import System.Path.NameManip
import System.Posix.Process

import Control.Concurrent.Chan.Len
import Pygmalion.Analysis.Extension
import Pygmalion.Analysis.Manager
import Pygmalion.Config
import Pygmalion.Core
import Pygmalion.Database.Manager
import Pygmalion.Log
import Pygmalion.RPC.Server

main :: IO ()
main = do
  cf <- getConfiguration
  initLogger (logLevel cf)
  nice 5
  ensureDB
  stopWatching <- newEmptyMVar
  port <- newEmptyMVar
  aChan <- newLenChan
  dbChan <- newLenChan
  dbQueryChan <- newLenChan
  logDebug "Launching database thread"
  dbThread <- asyncBound (runDatabaseManager dbChan dbQueryChan)
  link dbThread
  --let maxThreads = numCapabilities
  let maxThreads = 4 :: Int
  threads <- forM [1..maxThreads] $ \i -> do
    logDebug $ "Launching analysis thread #" ++ (show i)
    asyncBound (runAnalysisManager aChan dbChan dbQueryChan)
  mapM_ link threads
  rpcThread <- async (runRPCServer cf port aChan dbQueryChan)
  link rpcThread
  watchThread <- async (doWatch aChan stopWatching)
  link watchThread
  _ <- getLine
  cancel rpcThread
  logDebug "Terminated RPC thread."
  putMVar stopWatching ()
  wait watchThread
  logDebug "Terminated watch thread."
  forM_ threads $ \_ -> writeLenChan aChan ShutdownAnalysis  -- Signifies end of data.
  forM_ (zip threads [1..numCapabilities]) $ \(thread, i) -> do
    ensureNoException =<< waitCatch thread
    logDebug $ "Termination of thread #" ++ (show i)
  writeLenChan dbChan DBShutdown  -- Terminate the database thread.
  ensureNoException =<< waitCatch dbThread
  logDebug $ "Termination of database thread"

doWatch :: AnalysisChan -> MVar () -> IO ()
doWatch aChan stopWatching = do
  -- Restart every 10 minutes until a better fix is found. =(
{'fraction_non_alphanumeric': 0.08383233532934131, 'fraction_numerical': 0.0033532934131736527, 'mean_word_length': 4.149198520345252, 'pattern_counts': {'":': 0, '<': 35, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4375219', 'n_tokens_mistral': 1339, 'n_tokens_neox': 1250, 'n_words': 399}