<commit_before>import asyncio
import json
import os

from .cache import compute_key
from .error import PrintableError
from .edit_yaml import set_module_field_in_file
from . import imports
from .plugin import plugin_fetch, plugin_get_reup_fields
from . import scope


class Module:
    def __init__(self, name, type, default_rule, plugin_fields, yaml_name,
                 peru_file):
        self.name = name
        self.type = type
        self.default_rule = default_rule
        self.plugin_fields = plugin_fields
        self.yaml_name = yaml_name  # used by reup to edit the markup
        self.peru_file = peru_file  # for recursive module definitions

    @asyncio.coroutine
    def _get_base_tree(self, runtime):
        override_path = runtime.get_override(self.name)
        if override_path is not None:
            return self._get_override_tree(runtime, override_path)

        key = compute_key({
            'type': self.type,
            'plugin_fields': self.plugin_fields,
            'peru_file': self.peru_file,
        })
        # Use a lock to prevent the same module from being double fetched. The
        # lock is taken on the cache key, not the module itself, so two
        # different modules with identical fields will take the same lock and
        # avoid double fetching.
        cache_key_lock = runtime.cache_key_locks[key]
        with (yield from cache_key_lock):
            # Skip reading the cache if --no-cache is set. This is the only
            # place in the code we check that flag. Deterministic operations
            # like tree merging still get read from cache, because there's no
            # reason to redo them.
            if key in runtime.cache.keyval and not runtime.no_cache:
                return runtime.cache.keyval[key]
            with runtime.tmp_dir() as tmp_dir:
                yield from plugin_fetch(
                    runtime.get_plugin_context(), self.type,
                    self.plugin_fields, tmp_dir,
                    runtime.display.get_handle(self.name))
                tree = runtime.cache.import_tree(tmp_dir)
            # Note that we still *write* to cache even when --no-cache is True.
            # That way we avoid confusing results on subsequent syncs.
            runtime.cache.keyval[key] = tree
        return tree

    @asyncio.coroutine
    def get_tree(self, runtime):
        # TODO: memoize the tree or something
        base_tree = yield from self._get_base_tree(runtime)
        scope, _imports = yield from self.parse_peru_file(runtime)
        if not scope:
            return base_tree
        recursive_tree = yield from imports.get_imports_tree(
            runtime, scope, _imports, base_tree=base_tree)
        return recursive_tree

    @asyncio.coroutine
    def parse_peru_file(self, runtime):
        from . import parser  # avoid circular imports
        tree = yield from self._get_base_tree(runtime)
        cache_key = compute_key({
            'key_type': 'module_peru_file',
            'input_tree': tree,
            'file_name': self.peru_file,
        })
        if cache_key in runtime.cache.keyval:
            yaml = json.loads(runtime.cache.keyval[cache_key])
        else:
            try:
                yaml = runtime.cache.read_file(tree, self.peru_file)
            except FileNotFoundError:<commit_msg>decode YAML to a string before caching it<commit_after>import asyncio
import json
import os

from .cache import compute_key
from .error import PrintableError
from .edit_yaml import set_module_field_in_file
from . import imports
from .plugin import plugin_fetch, plugin_get_reup_fields
from . import scope


class Module:
    def __init__(self, name, type, default_rule, plugin_fields, yaml_name,
                 peru_file):
        self.name = name
        self.type = type
        self.default_rule = default_rule
        self.plugin_fields = plugin_fields
        self.yaml_name = yaml_name  # used by reup to edit the markup
        self.peru_file = peru_file  # for recursive module definitions

    @asyncio.coroutine
    def _get_base_tree(self, runtime):
        override_path = runtime.get_override(self.name)
        if override_path is not None:
            return self._get_override_tree(runtime, override_path)

        key = compute_key({
            'type': self.type,
            'plugin_fields': self.plugin_fields,
            'peru_file': self.peru_file,
        })
        # Use a lock to prevent the same module from being double fetched. The
        # lock is taken on the cache key, not the module itself, so two
        # different modules with identical fields will take the same lock and
        # avoid double fetching.
        cache_key_lock = runtime.cache_key_locks[key]
        with (yield from cache_key_lock):
            # Skip reading the cache if --no-cache is set. This is the only
            # place in the code we check that flag. Deterministic operations
            # like tree merging still get read from cache, because there's no
            # reason to redo them.
            if key in runtime.cache.keyval and not runtime.no_cache:
                return runtime.cache.keyval[key]
            with runtime.tmp_dir() as tmp_dir:
                yield from plugin_fetch(
                    runtime.get_plugin_context(), self.type,
                    self.plugin_fields, tmp_dir,
                    runtime.display.get_handle(self.name))
                tree = runtime.cache.import_tree(tmp_dir)
            # Note that we still *write* to cache even when --no-cache is True.
            # That way we avoid confusing results on subsequent syncs.
            runtime.cache.keyval[key] = tree
        return tree

    @asyncio.coroutine
    def get_tree(self, runtime):
        # TODO: memoize the tree or something
        base_tree = yield from self._get_base_tree(runtime)
        scope, _imports = yield from self.parse_peru_file(runtime)
        if not scope:
            return base_tree
        recursive_tree = yield from imports.get_imports_tree(
            runtime, scope, _imports, base_tree=base_tree)
        return recursive_tree

    @asyncio.coroutine
    def parse_peru_file(self, runtime):
        from . import parser  # avoid circular imports
        tree = yield from self._get_base_tree(runtime)
        cache_key = compute_key({
            'key_type': 'module_peru_file',
            'input_tree': tree,
            'file_name': self.peru_file,
        })
        if cache_key in runtime.cache.keyval:
            yaml = json.loads(runtime.cache.keyval[cache_key])
        else:
            try:
                yaml_bytes = runtime.cache.read_file(tree, self.peru_file)
                yaml = yaml_bytes.decode('utf8')
            except FileNotFoundError:
{'fraction_non_alphanumeric': 0.06675527987003396, 'fraction_numerical': 0.0001476886722788362, 'mean_word_length': 2.1883239171374766, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1946667', 'n_tokens_mistral': 1883, 'n_tokens_neox': 1837, 'n_words': 576}