<commit_before>import operator
from os.path import isdir, isfile, join, basename
from django.conf import settings
# Use the built-in version of scandir/walk if possible, otherwise
# use the scandir module version
try:
    from os import scandir, walk
except ImportError:
    from scandir import scandir, walk

from graphite.logger import log
from graphite.node import BranchNode, LeafNode
from graphite.readers import WhisperReader, GzippedWhisperReader, RRDReader
from graphite.util import find_escaped_pattern_fields
from graphite.finders.utils import BaseFinder

from . import fs_to_metric, get_real_metric_path, match_entries


class StandardFinder(BaseFinder):
    DATASOURCE_DELIMITER = '::RRD_DATASOURCE::'

    def __init__(self, directories=None):
        directories = directories or settings.STANDARD_DIRS
        self.directories = directories

    def find_nodes(self, query):
        clean_pattern = query.pattern.replace('\\', '')
        pattern_parts = clean_pattern.split('.')

        for root_dir in self.directories:
            for absolute_path in self._find_paths(root_dir, pattern_parts):
                if basename(absolute_path).startswith('.'):
                    continue

                if self.DATASOURCE_DELIMITER in basename(absolute_path):
                    (absolute_path, datasource_pattern) = absolute_path.rsplit(
                        self.DATASOURCE_DELIMITER, 1)
                else:
                    datasource_pattern = None

                relative_path = absolute_path[len(root_dir):].lstrip('/')
                metric_path = fs_to_metric(relative_path)
                real_metric_path = get_real_metric_path(
                    absolute_path, metric_path)

                metric_path_parts = metric_path.split('.')
                for field_index in find_escaped_pattern_fields(query.pattern):
                    metric_path_parts[field_index] = pattern_parts[field_index].replace(
                        '\\', '')
                metric_path = '.'.join(metric_path_parts)

                # Now we construct and yield an appropriate Node object
                if isdir(absolute_path):
                    yield BranchNode(metric_path)

                elif isfile(absolute_path):
                    if absolute_path.endswith(
                            '.wsp') and WhisperReader.supported:
                        reader = WhisperReader(absolute_path, real_metric_path)
                        yield LeafNode(metric_path, reader)<commit_msg>store tagged whisper files in a separate hierarchy under _tagged<commit_after>import operator
from os.path import isdir, isfile, join, basename
from django.conf import settings
from hashlib import sha256
# Use the built-in version of scandir/walk if possible, otherwise
# use the scandir module version
try:
    from os import scandir, walk
except ImportError:
    from scandir import scandir, walk

from graphite.logger import log
from graphite.node import BranchNode, LeafNode
from graphite.readers import WhisperReader, GzippedWhisperReader, RRDReader
from graphite.util import find_escaped_pattern_fields
from graphite.finders.utils import BaseFinder

from . import fs_to_metric, get_real_metric_path, match_entries


class StandardFinder(BaseFinder):
    DATASOURCE_DELIMITER = '::RRD_DATASOURCE::'

    def __init__(self, directories=None):
        directories = directories or settings.STANDARD_DIRS
        self.directories = directories

    def find_nodes(self, query):
        clean_pattern = query.pattern.replace('\\', '')

        # translate query pattern if it is tagged
        if ';' in query.pattern:
          hash = sha256(query.pattern).hexdigest()
          clean_pattern = '.'.join(['_tagged', hash[0:3], hash[3:6], query.pattern.replace('.', '-')])

        pattern_parts = clean_pattern.split('.')

        for root_dir in self.directories:
            for absolute_path in self._find_paths(root_dir, pattern_parts):
                if basename(absolute_path).startswith('.'):
                    continue

                if self.DATASOURCE_DELIMITER in basename(absolute_path):
                    (absolute_path, datasource_pattern) = absolute_path.rsplit(
                        self.DATASOURCE_DELIMITER, 1)
                else:
                    datasource_pattern = None

                relative_path = absolute_path[len(root_dir):].lstrip('/')
                metric_path = fs_to_metric(relative_path)
                real_metric_path = get_real_metric_path(
                    absolute_path, metric_path)

                metric_path_parts = metric_path.split('.')
                for field_index in find_escaped_pattern_fields(query.pattern):
                    metric_path_parts[field_index] = pattern_parts[field_index].replace(
                        '\\', '')
                metric_path = '.'.join(metric_path_parts)

                # if we're finding by tag, return the proper metric path
                if ';' in query.pattern:
                  metric_path = query.pattern

                # Now we construct and yield an appropriate Node object
                if isdir(absolute_path):
                    yield BranchNode(metric_path)

                elif isfile(absolute_path):
                    if absolute_path.endswith(
                            '.wsp') and WhisperReader.supported:
                        reader = WhisperReader(absolute_path, real_metric_path)
                        yield LeafNode(metric_path, reader)
{'fraction_non_alphanumeric': 0.07459970887918486, 'fraction_numerical': 0.002183406113537118, 'mean_word_length': 2.303485576923077, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5268793', 'n_tokens_mistral': 1530, 'n_tokens_neox': 1416, 'n_words': 356}