<commit_before>-- | Functions for file handling in the master node.

module FeedGipeda.Master.File
  ( writeBenchmarkCSV
  , isBenchmarkCSV
  , readBacklog
  , isBacklog
  , repoOfPath
  ) where


import           Control.Concurrent  (threadDelay)
import           Data.Char           (toLower)
import           Data.Functor
import           Data.List           (isSuffixOf)
import           Data.Maybe          (listToMaybe)
import           Data.Set            (Set)
import qualified Data.Set            as Set
import           FeedGipeda.GitShell (SHA)
import qualified FeedGipeda.GitShell as GitShell
import           FeedGipeda.Prelude
import           FeedGipeda.Repo     (Repo)
import qualified FeedGipeda.Repo     as Repo
import           System.Directory    (createDirectoryIfMissing, doesFileExist,
                                      getCurrentDirectory, getDirectoryContents)
import           System.FilePath     (dropFileName, makeRelative, normalise,
                                      splitDirectories, takeBaseName,
                                      takeBaseName, takeExtension, takeFileName,
                                      (<.>), (</>))


readBacklog :: Repo -> IO [SHA]
readBacklog repo = do
  backlog <- Repo.backlogFile repo
  maybe [] lines <$> readFileMaybe backlog


matchProjectRelativeDirectory :: [String] -> FilePath -> FilePath -> Bool
matchProjectRelativeDirectory subDirs cwd path =
  let
    splitNormalisedDirs :: [String]
    splitNormalisedDirs =
      splitDirectories (makeRelative cwd (normalise (dropFileName path)))
  in
    length splitNormalisedDirs == length subDirs + 1
     && subDirs `isSuffixOf` splitNormalisedDirs


isBacklog :: FilePath -> FilePath -> Bool
isBacklog cwd path =
  and
    [ map toLower (takeBaseName path) == "backlog"
    , map toLower (takeExtension path) == ".txt"
    , matchProjectRelativeDirectory ["site", "out"] cwd path
    ]


writeBenchmarkCSV :: Repo -> SHA -> String -> IO ()
writeBenchmarkCSV repo commit result = do
  cwd <- getCurrentDirectory
  resultsDir <- Repo.resultsDir repo
  createDirectoryIfMissing True resultsDir
  threadDelay 100000
  writeFile (resultsDir </> commit <.> "csv") result


isBenchmarkCSV :: FilePath -> FilePath -> Bool
isBenchmarkCSV cwd path =
  map toLower (takeExtension path) == ".csv"
   && matchProjectRelativeDirectory ["site", "out", "results"] cwd path
<commit_msg>1 ms should be enough time for the fsnotify to catch up<commit_after>-- | Functions for file handling in the master node.

module FeedGipeda.Master.File
  ( writeBenchmarkCSV
  , isBenchmarkCSV
  , readBacklog
  , isBacklog
  , repoOfPath
  ) where


import           Control.Concurrent  (threadDelay)
import           Data.Char           (toLower)
import           Data.Functor
import           Data.List           (isSuffixOf)
import           Data.Maybe          (listToMaybe)
import           Data.Set            (Set)
import qualified Data.Set            as Set
import           FeedGipeda.GitShell (SHA)
import qualified FeedGipeda.GitShell as GitShell
import           FeedGipeda.Prelude
import           FeedGipeda.Repo     (Repo)
import qualified FeedGipeda.Repo     as Repo
import           System.Directory    (createDirectoryIfMissing, doesFileExist,
                                      getCurrentDirectory, getDirectoryContents)
import           System.FilePath     (dropFileName, makeRelative, normalise,
                                      splitDirectories, takeBaseName,
                                      takeBaseName, takeExtension, takeFileName,
                                      (<.>), (</>))


readBacklog :: Repo -> IO [SHA]
readBacklog repo = do
  backlog <- Repo.backlogFile repo
  maybe [] lines <$> readFileMaybe backlog


matchProjectRelativeDirectory :: [String] -> FilePath -> FilePath -> Bool
matchProjectRelativeDirectory subDirs cwd path =
  let
    splitNormalisedDirs :: [String]
    splitNormalisedDirs =
      splitDirectories (makeRelative cwd (normalise (dropFileName path)))
  in
    length splitNormalisedDirs == length subDirs + 1
     && subDirs `isSuffixOf` splitNormalisedDirs


isBacklog :: FilePath -> FilePath -> Bool
isBacklog cwd path =
  and
    [ map toLower (takeBaseName path) == "backlog"
    , map toLower (takeExtension path) == ".txt"
    , matchProjectRelativeDirectory ["site", "out"] cwd path
    ]


writeBenchmarkCSV :: Repo -> SHA -> String -> IO ()
writeBenchmarkCSV repo commit result = do
  cwd <- getCurrentDirectory
  resultsDir <- Repo.resultsDir repo
  createDirectoryIfMissing True resultsDir
  threadDelay 1000
  writeFile (resultsDir </> commit <.> "csv") result


isBenchmarkCSV :: FilePath -> FilePath -> Bool
isBenchmarkCSV cwd path =
  map toLower (takeExtension path) == ".csv"
   && matchProjectRelativeDirectory ["site", "out", "results"] cwd path

{'fraction_non_alphanumeric': 0.08102521703183134, 'fraction_numerical': 0.002687060768912774, 'mean_word_length': 2.471305595408895, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1529133', 'n_tokens_mistral': 1387, 'n_tokens_neox': 1328, 'n_words': 373}