<commit_before>from fnmatch import fnmatch
from os import stat, walk
from os.path import abspath, exists, join
import time

from checks import AgentCheck


class DirectoryCheck(AgentCheck):
    """This check is for monitoring and reporting metrics on the files for a provided directory

    WARNING: the user/group that dd-agent runs as must have access to stat the files in the desired directory

    Config options:
        "directory" - string, the directory to gather stats for. required
        "name" - string, the name to use when tagging the metrics. defaults to the "directory"
        "pattern" - string, the `fnmatch` pattern to use when reading the "directory"'s files. default "*"
        "recursive" - boolean, when true the stats will recurse into directories. default False
    """
    def check(self, instance):
        if "directory" not in instance:
            raise Exception('DirectoryCheck: missing "directory" in config')

        directory = instance["directory"]
        abs_directory = abspath(directory)
        name = instance.get("name") or directory
        pattern = instance.get("pattern") or "*"
        recursive = instance.get("recursive") or False

        if not exists(abs_directory):
            raise Exception("DirectoryCheck: the directory (%s) does not exist" % abs_directory)

        self._get_stats(abs_directory, name, pattern, recursive)

    def _get_stats(self, directory, name, pattern, recursive):
        tags = ["name:%s" % name]
        directory_bytes = 0
        directory_files = 0
        for root, dirs, files in walk(directory):
            for filename in files:
                filename = join(root, filename)
                # check if it passes our filter
                if not fnmatch(filename, pattern):
                    continue
                try:
                    file_stat = stat(filename)

                except OSError, ose:
                    self.warning("DirectoryCheck: could not stat file %s - %s" % (filename, ose))
                else:
                    directory_files += 1
                    directory_bytes += file_stat.st_size
                    # file specific metrics
                    self.histogram("directory.file.bytes", file_stat.st_size, tags=tags)
                    self.histogram("directory.file.modified_sec_ago", time.time() - file_stat.st_mtime, tags=tags)
                    self.histogram("directory.file.created_sec_ago", time.time() - file_stat.st_ctime, tags=tags)

            # os.walk gives us all sub-directories and their files
            # if we do not want to do this recursively and just want
            # the top level directory we gave it, then break
            if not recursive:
                break

        # number of files
        self.gauge("directory.files", directory_files, tags=tags)
        # total file size
        self.gauge("directory.bytes", directory_bytes, tags=tags)<commit_msg>prefix directory check metric names with system.disk<commit_after>from fnmatch import fnmatch
from os import stat, walk
from os.path import abspath, exists, join
import time

from checks import AgentCheck


class DirectoryCheck(AgentCheck):
    """This check is for monitoring and reporting metrics on the files for a provided directory

    WARNING: the user/group that dd-agent runs as must have access to stat the files in the desired directory

    Config options:
        "directory" - string, the directory to gather stats for. required
        "name" - string, the name to use when tagging the metrics. defaults to the "directory"
        "pattern" - string, the `fnmatch` pattern to use when reading the "directory"'s files. default "*"
        "recursive" - boolean, when true the stats will recurse into directories. default False
    """
    def check(self, instance):
        if "directory" not in instance:
            raise Exception('DirectoryCheck: missing "directory" in config')

        directory = instance["directory"]
        abs_directory = abspath(directory)
        name = instance.get("name") or directory
        pattern = instance.get("pattern") or "*"
        recursive = instance.get("recursive") or False

        if not exists(abs_directory):
            raise Exception("DirectoryCheck: the directory (%s) does not exist" % abs_directory)

        self._get_stats(abs_directory, name, pattern, recursive)

    def _get_stats(self, directory, name, pattern, recursive):
        tags = ["name:%s" % name]
        directory_bytes = 0
        directory_files = 0
        for root, dirs, files in walk(directory):
            for filename in files:
                filename = join(root, filename)
                # check if it passes our filter
                if not fnmatch(filename, pattern):
                    continue
                try:
                    file_stat = stat(filename)

                except OSError, ose:
                    self.warning("DirectoryCheck: could not stat file %s - %s" % (filename, ose))
                else:
                    directory_files += 1
                    directory_bytes += file_stat.st_size
                    # file specific metrics
                    self.histogram("system.disk.directory.file.bytes", file_stat.st_size, tags=tags)
                    self.histogram("system.disk.directory.file.modified_sec_ago", time.time() - file_stat.st_mtime, tags=tags)
                    self.histogram("system.disk.directory.file.created_sec_ago", time.time() - file_stat.st_ctime, tags=tags)

            # os.walk gives us all sub-directories and their files
            # if we do not want to do this recursively and just want
            # the top level directory we gave it, then break
            if not recursive:
                break

        # number of files
        self.gauge("system.disk.directory.files", directory_files, tags=tags)
        # total file size
        self.gauge("system.disk.directory.bytes", directory_bytes, tags=tags)
{'fraction_non_alphanumeric': 0.08128037539802245, 'fraction_numerical': 0.001005530417295123, 'mean_word_length': 2.247007616974973, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6398188', 'n_tokens_mistral': 1535, 'n_tokens_neox': 1447, 'n_words': 539}