<commit_before>import fnmatch
import json
import os
import multiprocessing
import re
import sys

from collections import OrderedDict

try:
    from colorama import Fore
    from colorama import Style
except ImportError:
    class EmptyStringAttributes:
        def __getattr__(self, name):
            return ''
    Fore = EmptyStringAttributes()
    Style = EmptyStringAttributes()

from .process import command_output
from .library import Library
from .universal_binary import UniversalBinary
from .platform import available_platforms, host_platform
from .generator import available_generators
from .target import Target
from .cd import current_directory


class Needy:
    def __init__(self, path, parameters):
        self.__path = path if os.path.isabs(path) else os.path.normpath(os.path.join(current_directory(), path))
        self.__parameters = parameters

        self.__needs_directory = os.path.join(self.__path, 'needs')
        self.__needs_file = self.find_needs_file(self.__path)

        directory = self.__path
        while directory:
            if self.find_needs_file(directory):
                self.__needs_directory = os.path.join(directory, 'needs')
            directory = os.path.dirname(directory)
            if directory == os.sep:
                break

    def path(self):
        return self.__path

    def needs_file(self):
        return self.__needs_file

    @staticmethod
    def find_needs_file(directory):
        ret = None
        for name in ['needs.json', 'needs.yaml']:
            path = os.path.join(directory, name)
            if os.path.isfile(path):
                if ret:
                    raise RuntimeError('More than one needs file is present.')
                ret = path
        return ret

    def needs_configuration(self, target=None):
        configuration = ''
        with open(self.needs_file(), 'r') as needs_file:
            configuration = needs_file.read()

        try:
            from jinja2 import Environment, PackageLoader
            env = Environment()
            env.filters['dirname'] = os.path.dirname
            template = env.from_string(configuration)
            configuration = template.render(
                platform=target.platform.identifier() if target else None,
                architecture=target.architecture if target else None,
                host_platform=host_platform().identifier(),
                needs_file=self.needs_file()
            )
        except ImportError:
            if re.compile('{%.*%}').search(configuration) or re.compile('{{.*}}').search(configuration) or re.compile('{#.*#}').search(configuration):
                raise RuntimeError('The needs file appears to contain Jinja templating. Please install the jinja2 Python package.')

        name, extension = os.path.splitext(self.needs_file())

        if extension == '.json':
            return json.loads(configuration, object_pairs_hook=OrderedDict)

        if extension == '.yaml':
            try:
                import yaml
                class OrderedLoader(yaml.SafeLoader):
                    pass
                def construct_mapping(loader, node):
                    loader.flatten_mapping(node)
                    return OrderedDict(loader.construct_pairs(node))
                OrderedLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping)
                return yaml.load(configuration, OrderedLoader)
            except ImportError:
                raise RuntimeError('The needs are defined in a YAML file. Please install the pyyaml Python package.')

            return None

        raise NotImplementedError('file parsing not implemented')
<commit_msg>Add needs_directory for Jinja templates<commit_after>import fnmatch
import json
import os
import multiprocessing
import re
import sys

from collections import OrderedDict

try:
    from colorama import Fore
    from colorama import Style
except ImportError:
    class EmptyStringAttributes:
        def __getattr__(self, name):
            return ''
    Fore = EmptyStringAttributes()
    Style = EmptyStringAttributes()

from .process import command_output
from .library import Library
from .universal_binary import UniversalBinary
from .platform import available_platforms, host_platform
from .generator import available_generators
from .target import Target
from .cd import current_directory


class Needy:
    def __init__(self, path, parameters):
        self.__path = path if os.path.isabs(path) else os.path.normpath(os.path.join(current_directory(), path))
        self.__parameters = parameters

        self.__needs_directory = os.path.join(self.__path, 'needs')
        self.__needs_file = self.find_needs_file(self.__path)

        directory = self.__path
        while directory:
            if self.find_needs_file(directory):
                self.__needs_directory = os.path.join(directory, 'needs')
            directory = os.path.dirname(directory)
            if directory == os.sep:
                break

    def path(self):
        return self.__path

    def needs_file(self):
        return self.__needs_file

    @staticmethod
    def find_needs_file(directory):
        ret = None
        for name in ['needs.json', 'needs.yaml']:
            path = os.path.join(directory, name)
            if os.path.isfile(path):
                if ret:
                    raise RuntimeError('More than one needs file is present.')
                ret = path
        return ret

    def needs_configuration(self, target=None):
        configuration = ''
        with open(self.needs_file(), 'r') as needs_file:
            configuration = needs_file.read()

        try:
            from jinja2 import Environment, PackageLoader
            env = Environment()
            env.filters['dirname'] = os.path.dirname
            template = env.from_string(configuration)
            configuration = template.render(
                platform=target.platform.identifier() if target else None,
                architecture=target.architecture if target else None,
                host_platform=host_platform().identifier(),
                needs_file=self.needs_file(),
                needs_directory=self.needs_directory()
            )
        except ImportError:
            if re.compile('{%.*%}').search(configuration) or re.compile('{{.*}}').search(configuration) or re.compile('{#.*#}').search(configuration):
                raise RuntimeError('The needs file appears to contain Jinja templating. Please install the jinja2 Python package.')

        name, extension = os.path.splitext(self.needs_file())

        if extension == '.json':
            return json.loads(configuration, object_pairs_hook=OrderedDict)

        if extension == '.yaml':
            try:
                import yaml
                class OrderedLoader(yaml.SafeLoader):
                    pass
                def construct_mapping(loader, node):
                    loader.flatten_mapping(node)
                    return OrderedDict(loader.construct_pairs(node))
                OrderedLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping)
                return yaml.load(configuration, OrderedLoader)
            except ImportError:
                raise RuntimeError('The needs are defined in a YAML file. Please install the pyyaml Python package.')

            return None

        raise NotImplementedError('file parsing not implemented')

{'fraction_non_alphanumeric': 0.08364561027837258, 'fraction_numerical': 0.0005353319057815846, 'mean_word_length': 2.328730512249443, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6561394', 'n_tokens_mistral': 1926, 'n_tokens_neox': 1811, 'n_words': 507}