<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
# searcher.py
#
# class Searcher: executes a file search
#
###############################################################################
from collections import deque
from io import StringIO
import os

TARFILE_MODULE_AVAILABLE = True
ZIPFILE_MODULE_AVAILABLE = True

try:
    import tarfile
except ImportError as ie:
    print('tarfile not imported: {0!s}'.format(ie))
    TARFILE_MODULE_AVAILABLE = False
try:
    import zipfile
except ImportError as ie:
    print('zipfile not imported: {0!s}'.format(ie))
    ZIPFILE_MODULE_AVAILABLE = False

from .common import log
from .filetypes import FileType, FileTypes
from .fileutil import FileUtil
from .searchfile import SearchFile
from .searchresult import SearchResult


class Searcher(object):
    """a class to search files"""

    def __init__(self, settings, **kargs):
        self.settings = settings
        self.validate_settings()
        self.filetypes = FileTypes()
        self.results = []
        self.patterndict = {}
        self.filedict = {}
        self.rescounts = {}
        self.__dict__.update(kargs)

    def validate_settings(self):
        """Assert required settings in SearchSettings instance"""
        assert self.settings.startpath, 'Startpath not defined'
        assert os.path.exists(self.settings.startpath), 'Startpath not found'
        assert self.settings.searchpatterns, 'No search patterns defined'

    def is_search_dir(self, d):
        path_elems = [p for p in d.split(os.sep) if p not in FileUtil.DOT_DIRS]
        if self.settings.excludehidden:
            for p in path_elems:
                if FileUtil.is_hidden(p):
                    return False
        if self.settings.in_dirpatterns and \
            not any_matches_any_pattern(path_elems, self.settings.in_dirpatterns):
            return False
        if self.settings.out_dirpatterns and \
            any_matches_any_pattern(path_elems, self.settings.out_dirpatterns):
            return False
        return True

    def is_archive_search_file(self, f):
        ext = FileUtil.get_extension(f)
        if self.settings.in_archiveextensions and \
            ext not in self.settings.in_archiveextensions:
            return False<commit_msg>Check for readable startpath<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
# searcher.py
#
# class Searcher: executes a file search
#
###############################################################################
from collections import deque
from io import StringIO
import os

TARFILE_MODULE_AVAILABLE = True
ZIPFILE_MODULE_AVAILABLE = True

try:
    import tarfile
except ImportError as ie:
    print('tarfile not imported: {0!s}'.format(ie))
    TARFILE_MODULE_AVAILABLE = False
try:
    import zipfile
except ImportError as ie:
    print('zipfile not imported: {0!s}'.format(ie))
    ZIPFILE_MODULE_AVAILABLE = False

from .common import log
from .filetypes import FileType, FileTypes
from .fileutil import FileUtil
from .searchfile import SearchFile
from .searchresult import SearchResult


class Searcher(object):
    """a class to search files"""

    def __init__(self, settings, **kargs):
        self.settings = settings
        self.validate_settings()
        self.filetypes = FileTypes()
        self.results = []
        self.patterndict = {}
        self.filedict = {}
        self.rescounts = {}
        self.__dict__.update(kargs)

    def validate_settings(self):
        """Assert required settings in SearchSettings instance"""
        assert self.settings.startpath, 'Startpath not defined'
        assert os.path.exists(self.settings.startpath), 'Startpath not found'
        assert os.access(self.settings.startpath, os.R_OK), 'Startpath not readable'
        assert self.settings.searchpatterns, 'No search patterns defined'

    def is_search_dir(self, d):
        path_elems = [p for p in d.split(os.sep) if p not in FileUtil.DOT_DIRS]
        if self.settings.excludehidden:
            for p in path_elems:
                if FileUtil.is_hidden(p):
                    return False
        if self.settings.in_dirpatterns and \
            not any_matches_any_pattern(path_elems, self.settings.in_dirpatterns):
            return False
        if self.settings.out_dirpatterns and \
            any_matches_any_pattern(path_elems, self.settings.out_dirpatterns):
            return False
        return True

    def is_archive_search_file(self, f):
        ext = FileUtil.get_extension(f)
        if self.settings.in_archiveextensions and \
            ext not in self.settings.in_archiveextensions:
            return False
{'fraction_non_alphanumeric': 0.1442995372318048, 'fraction_numerical': 0.0012620950778291964, 'mean_word_length': 3.0571672354948807, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '112348', 'n_tokens_mistral': 1329, 'n_tokens_neox': 1298, 'n_words': 350}