<commit_before>module HsSearch.Searcher
    (
      doSearch
    , doSearchFiles
    , getSearchDirs
    , getSearchFiles
    , isArchiveSearchFile
    , isSearchDir
    , isSearchFile
    ) where

import Control.Monad (liftM)
import qualified Data.ByteString as B
import qualified Data.ByteString.Char8 as BC
import Data.Maybe (catMaybes)
import Text.Regex.PCRE

import HsSearch.FileTypes
import HsSearch.FileUtil
import HsSearch.SearchFile
import HsSearch.SearchResult
import HsSearch.SearchSettings


isSearchDir :: SearchSettings -> FilePath -> Bool
isSearchDir settings d = and $ map ($d) tests
  where tests :: [FilePath -> Bool]
        tests = [ (\x -> length inPatterns == 0
                         || any (\p -> x =~ p :: Bool) inPatterns)
                , (\x -> length outPatterns == 0
                         || all (\p -> not $ x =~ p :: Bool) outPatterns)
                , (\x -> not (isHiddenFilePath x) || includeHidden)
                ]
        inPatterns = inDirPatterns settings
        outPatterns = outDirPatterns settings
        includeHidden = not $ excludeHidden settings

getSearchDirs :: SearchSettings -> IO [FilePath]
getSearchDirs settings = do
  isStartPathDir <- isDirectory $ startPath settings
  searchDirs <- do if isStartPathDir
                     then do
                       ds <- getDirectories
                       return ds
                     else return [getParentPath (startPath settings)]
  return searchDirs
  where getDirectories :: IO [FilePath]
        getDirectories = do
          dirs <- getRecursiveDirectories $ startPath settings
          return $ filter (isSearchDir settings) $ [startPath settings] ++ dirs

isSearchFile :: SearchSettings -> FilePath -> Bool
isSearchFile settings fp = and $ map ($fp) tests
  where tests :: [FilePath -> Bool]
        tests = [ (\x -> length inExts == 0
                         || hasInExt x)
                , (\x -> length outExts == 0
                         || not (hasOutExt x))
                , (\x -> length inPatterns == 0
                         || any (\p -> x =~ p :: Bool) inPatterns)
                , (\x -> length outPatterns == 0
                         || all (\p -> not $ x =~ p :: Bool) outPatterns)
                , (\x -> not (isHiddenFilePath x) || includeHidden)
                ]
        inExts = inExtensions settings
        hasInExt f | null inExts = True
                   | otherwise   = any (hasExtension f) inExts
        outExts = outExtensions settings
        hasOutExt f | null outExts = False
                    | otherwise    = any (hasExtension f) outExts
        inPatterns = inFilePatterns settings
        outPatterns = outFilePatterns settings
        includeHidden = not $ excludeHidden settings

isArchiveSearchFile :: SearchSettings -> FilePath -> Bool
isArchiveSearchFile settings fp = and $ map ($fp) tests
  where tests :: [FilePath -> Bool]
        tests = [ (\x -> length inExts == 0
                         || hasInExt x)
                , (\x -> length outExts == 0
                         || not (hasOutExt x))
                , (\x -> length inPatterns == 0
                         || any (\p -> x =~ p :: Bool) inPatterns)
                , (\x -> length outPatterns == 0
                         || all (\p -> not $ x =~ p :: Bool) outPatterns)
                , (\x -> not (isHiddenFilePath x) || includeHidden)
                ]
        inExts = inArchiveExtensions settings
        hasInExt f | null inExts = True
                   | otherwise   = any (hasExtension f) inExts
        outExts = outArchiveExtensions settings
        hasOutExt f | null outExts = False
                    | otherwise    = any (hasExtension f) outExts
        inPatterns = inArchiveFilePatterns settings
        outPatterns = outArchiveFilePatterns settings
        includeHidden = not $ excludeHidden settings

getSearchFiles :: SearchSettings -> [FilePath] -> IO [FilePath]
getSearchFiles settings dirs = do
  isStartPathDir <- isDirectory $ startPath settings
  files <- do if isStartPathDir
                then do
                  fs <- concat `liftM` mapM getDirectoryFiles dirs
                  return fs
                else return [startPath settings]
  fileTypes <- getFileTypes files
  let makeSearchFile (f,t) = SearchFile { searchFileContainers=[]
                                        , searchFilePath=f
                                        , searchFileType=t }
  let filesWithTypes = map makeSearchFile (zip files fileTypes)
  let searchableFiles = filter isSearchableFile filesWithTypes
  let includeArchiveFile sf = (searchArchives settings) &&
                              isArchiveSearchFile settings (searchFilePath sf)
  let includeFile sf = (not (archivesOnly settings)) &&
                       isSearchFile settings (searchFilePath sf)
  let filterFile sf | isArchiveFile sf = includeArchiveFile sf
                    | otherwise        = includeFile sf
  return $ map searchFilePath (filter filterFile searchableFiles)

searchBinaryFile :: SearchSettings -> FilePath -> IO [SearchResult]
searchBinaryFile settings f = do
  blobEither <- getFileByteString f
  case blobEither of
    (Left _) -> return [] -- todo: figure out to relay error
    (Right blob) -> return $ addFilePath (searchBlob settings blob)
  where addFilePath = map (\r -> r {filePath=f})

searchBlob :: SearchSettings -> B.ByteString -> [SearchResult]
searchBlob settings blob =
  concat $ map (searchBlobForPattern blob) (searchPatterns settings)

searchBlobForPattern :: B.ByteString -> String -> [SearchResult]
searchBlobForPattern blob pattern = if hasMatch
                                      then [blobResult pattern]
                                      else []
  where hasMatch = blob =~ pattern :: Bool
        blobResult p = blankSearchResult { searchPattern=p
                                         , lineNum=0
                                         , matchStartIndex=0
                                         , matchEndIndex=0
                                         , line=B.empty
                                         }<commit_msg>Moved filterFile out of getSearchFiles in haskell<commit_after>module HsSearch.Searcher
    (
      doSearch
    , doSearchFiles
    , getSearchDirs
    , getSearchFiles
    , isArchiveSearchFile
    , isSearchDir
    , isSearchFile
    ) where

import Control.Monad (liftM)
import qualified Data.ByteString as B
import qualified Data.ByteString.Char8 as BC
import Data.Maybe (catMaybes)
import Text.Regex.PCRE

import HsSearch.FileTypes
import HsSearch.FileUtil
import HsSearch.SearchFile
import HsSearch.SearchResult
import HsSearch.SearchSettings


isSearchDir :: SearchSettings -> FilePath -> Bool
isSearchDir settings d = and $ map ($d) tests
  where tests :: [FilePath -> Bool]
        tests = [ (\x -> length inPatterns == 0
                         || any (\p -> x =~ p :: Bool) inPatterns)
                , (\x -> length outPatterns == 0
                         || all (\p -> not $ x =~ p :: Bool) outPatterns)
                , (\x -> not (isHiddenFilePath x) || includeHidden)
                ]
        inPatterns = inDirPatterns settings
        outPatterns = outDirPatterns settings
        includeHidden = not $ excludeHidden settings

getSearchDirs :: SearchSettings -> IO [FilePath]
getSearchDirs settings = do
  isStartPathDir <- isDirectory $ startPath settings
  searchDirs <- do if isStartPathDir
                     then do
                       ds <- getDirectories
                       return ds
                     else return [getParentPath (startPath settings)]
  return searchDirs
  where getDirectories :: IO [FilePath]
        getDirectories = do
          dirs <- getRecursiveDirectories $ startPath settings
          return $ filter (isSearchDir settings) $ [startPath settings] ++ dirs

isSearchFile :: SearchSettings -> FilePath -> Bool
isSearchFile settings fp = and $ map ($fp) tests
  where tests :: [FilePath -> Bool]
        tests = [ (\x -> length inExts == 0
                         || hasInExt x)
                , (\x -> length outExts == 0
                         || not (hasOutExt x))
                , (\x -> length inPatterns == 0
                         || any (\p -> x =~ p :: Bool) inPatterns)
                , (\x -> length outPatterns == 0
                         || all (\p -> not $ x =~ p :: Bool) outPatterns)
                , (\x -> not (isHiddenFilePath x) || includeHidden)
                ]
        inExts = inExtensions settings
        hasInExt f | null inExts = True
                   | otherwise   = any (hasExtension f) inExts
        outExts = outExtensions settings
        hasOutExt f | null outExts = False
                    | otherwise    = any (hasExtension f) outExts
        inPatterns = inFilePatterns settings
        outPatterns = outFilePatterns settings
        includeHidden = not $ excludeHidden settings

isArchiveSearchFile :: SearchSettings -> FilePath -> Bool
isArchiveSearchFile settings fp = and $ map ($fp) tests
  where tests :: [FilePath -> Bool]
        tests = [ (\x -> length inExts == 0
                         || hasInExt x)
                , (\x -> length outExts == 0
                         || not (hasOutExt x))
                , (\x -> length inPatterns == 0
                         || any (\p -> x =~ p :: Bool) inPatterns)
                , (\x -> length outPatterns == 0
                         || all (\p -> not $ x =~ p :: Bool) outPatterns)
                , (\x -> not (isHiddenFilePath x) || includeHidden)
                ]
        inExts = inArchiveExtensions settings
        hasInExt f | null inExts = True
                   | otherwise   = any (hasExtension f) inExts
        outExts = outArchiveExtensions settings
        hasOutExt f | null outExts = False
                    | otherwise    = any (hasExtension f) outExts
        inPatterns = inArchiveFilePatterns settings
        outPatterns = outArchiveFilePatterns settings
        includeHidden = not $ excludeHidden settings

filterFile :: SearchSettings -> SearchFile -> Bool
filterFile settings sf | isArchiveFile sf = includeArchiveFile sf
                       | otherwise        = includeFile sf
  where includeArchiveFile f = (searchArchives settings) &&
                               isArchiveSearchFile settings (searchFilePath f)
        includeFile f = (not (archivesOnly settings)) &&
                        isSearchFile settings (searchFilePath f)

getSearchFiles :: SearchSettings -> [FilePath] -> IO [FilePath]
getSearchFiles settings dirs = do
  isStartPathDir <- isDirectory $ startPath settings
  files <- do if isStartPathDir
                then do
                  fs <- concat `liftM` mapM getDirectoryFiles dirs
                  return fs
                else return [startPath settings]
  fileTypes <- getFileTypes files
  let makeSearchFile (f,t) = SearchFile { searchFileContainers=[]
                                        , searchFilePath=f
                                        , searchFileType=t }
  let filesWithTypes = map makeSearchFile (zip files fileTypes)
  let searchableFiles = filter isSearchableFile filesWithTypes
  return $ map searchFilePath (filter (filterFile settings) searchableFiles)

searchBinaryFile :: SearchSettings -> FilePath -> IO [SearchResult]
searchBinaryFile settings f = do
  blobEither <- getFileByteString f
  case blobEither of
    (Left _) -> return [] -- todo: figure out to relay error
    (Right blob) -> return $ addFilePath (searchBlob settings blob)
  where addFilePath = map (\r -> r {filePath=f})

searchBlob :: SearchSettings -> B.ByteString -> [SearchResult]
searchBlob settings blob =
  concat $ map (searchBlobForPattern blob) (searchPatterns settings)

searchBlobForPattern :: B.ByteString -> String -> [SearchResult]
searchBlobForPattern blob pattern = if hasMatch
                                      then [blobResult pattern]
                                      else []
  where hasMatch = blob =~ pattern :: Bool
        blobResult p = blankSearchResult { searchPattern=p
                                         , lineNum=0
                                         , matchStartIndex=0
                                         , matchEndIndex=0
                                         , line=B.empty
                                         }
{'fraction_non_alphanumeric': 0.08119277206061097, 'fraction_numerical': 0.0022688598979013048, 'mean_word_length': 1.7697486535008977, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 45, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7576974', 'n_tokens_mistral': 3145, 'n_tokens_neox': 2967, 'n_words': 970}