<commit_before>import os
import shlex
from argparse import ArgumentParser

from dodo_commands.dependencies import yaml_round_trip_load
from dodo_commands.framework.command_map import CommandMapItem
from dodo_commands.framework.config import Key, expand_keys
from dodo_commands.framework.singleton import Dodo
from dodo_commands.framework.util import chop


class YamlCommandMapItem(CommandMapItem):
    def __init__(self, group, filename):
        super(YamlCommandMapItem, self).__init__(group, filename, 'yaml')


class YamlCommandHandler:
    def add_commands_to_map(self, command_dirs, file_map, command_map):
        for command_dir, files in file_map.items():
            for file in files:
                command_name, ext = os.path.splitext(os.path.basename(file))
                prefix = 'dodo.'
                if ext == '.yaml' and command_name.startswith(prefix):
                    command_name = command_name[len(prefix):]
                    with open(file) as ifs:
                        data = ruamel.yaml_round_trip_load(ifs.read())
                        for command_name in data.keys():
                            command_map[command_name] = YamlCommandMapItem(
                                group=os.path.basename(command_dir),
                                filename=file)

    def _expand(self, x):
        return os.path.expanduser(
            os.path.expandvars(expand_keys(Dodo.get_config(), x)))

    def execute(self, command_map_item, command_name):
        with open(command_map_item.filename) as ifs:
            data = yaml_round_trip_load(ifs.read())

        description = None
        for key, val in data[command_name].items():
            if key == '_description':
                description = val
<commit_msg>Fix import in yaml command handler<commit_after>import os
import shlex
from argparse import ArgumentParser

from dodo_commands.dependencies import yaml_round_trip_load
from dodo_commands.framework.command_map import CommandMapItem
from dodo_commands.framework.config import Key, expand_keys
from dodo_commands.framework.singleton import Dodo
from dodo_commands.framework.util import chop


class YamlCommandMapItem(CommandMapItem):
    def __init__(self, group, filename):
        super(YamlCommandMapItem, self).__init__(group, filename, 'yaml')


class YamlCommandHandler:
    def add_commands_to_map(self, command_dirs, file_map, command_map):
        for command_dir, files in file_map.items():
            for file in files:
                command_name, ext = os.path.splitext(os.path.basename(file))
                prefix = 'dodo.'
                if ext == '.yaml' and command_name.startswith(prefix):
                    command_name = command_name[len(prefix):]
                    with open(file) as ifs:
                        data = yaml_round_trip_load(ifs.read())
                        for command_name in data.keys():
                            command_map[command_name] = YamlCommandMapItem(
                                group=os.path.basename(command_dir),
                                filename=file)

    def _expand(self, x):
        return os.path.expanduser(
            os.path.expandvars(expand_keys(Dodo.get_config(), x)))

    def execute(self, command_map_item, command_name):
        with open(command_map_item.filename) as ifs:
            data = yaml_round_trip_load(ifs.read())

        description = None
        for key, val in data[command_name].items():
            if key == '_description':
                description = val

{'fraction_non_alphanumeric': 0.08243626062322947, 'fraction_numerical': 0.0, 'mean_word_length': 2.411594202898551, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2972023', 'n_tokens_mistral': 996, 'n_tokens_neox': 950, 'n_words': 225}