<commit_before>import sys
import yaml
from yaml.composer import Composer
from lingua.extractors import (
    register_extractor,
    Message,
)


class YafowilYamlExtractor(object):

    def __call__(self, filename, options):
        self.filename = filename
        self.messages = list()
        self.walk(filename)
        return self.messages

    def walk(self, filename):
        loader = yaml.Loader(open(filename).read())
        def compose_node(parent, index):
            node = Composer.compose_node(loader, parent, index)
            self.parse_message(node, loader.line)
            return node
        loader.compose_node = compose_node
        loader.get_single_data()

    def parse_message(self, node, lineno):
        value = node.value
        if not isinstance(value, basestring):
            return
        if value.startswith('i18n:'):
            parts = value.split(":")
            if len(parts) > 3:
                msg = 'to many : in {0}'.format(value)
                sys.stderr.write(msg)
                return
            if len(parts) == 2:
                message = Message(
                    None, parts[1], None, [], u'', u'',
                    (self.filename, lineno),
                )
            else:
                message = Message(
                    None, parts[1], None, [], parts[2], u'',
                    (self.filename, lineno),
                )
            self.messages.append(message)


@register_extractor('yafowil_yaml', ['.yaml', '.yml'])
def extract_yafowil_yaml(filename, options):
    extractor = YafowilYamlExtractor()
    return extractor(filename, options)<commit_msg>add Default: to comment<commit_after>import sys
import yaml
from yaml.composer import Composer
from lingua.extractors import (
    register_extractor,
    Message,
)


class YafowilYamlExtractor(object):

    def __call__(self, filename, options):
        self.filename = filename
        self.messages = list()
        self.walk(filename)
        return self.messages

    def walk(self, filename):
        loader = yaml.Loader(open(filename).read())
        def compose_node(parent, index):
            node = Composer.compose_node(loader, parent, index)
            self.parse_message(node, loader.line)
            return node
        loader.compose_node = compose_node
        loader.get_single_data()

    def parse_message(self, node, lineno):
        value = node.value
        if not isinstance(value, basestring):
            return
        if value.startswith('i18n:'):
            parts = value.split(":")
            if len(parts) > 3:
                msg = 'to many : in {0}'.format(value)
                sys.stderr.write(msg)
                return
            if len(parts) == 2:
                message = Message(
                    None, parts[1], None, [], u'', u'',
                    (self.filename, lineno),
                )
            else:
                message = Message(
                    None, parts[1], None, [], u'Default: ' + parts[2], u'',
                    (self.filename, lineno),
                )
            self.messages.append(message)


@register_extractor('yafowil_yaml', ['.yaml', '.yml'])
def extract_yafowil_yaml(filename, options):
    extractor = YafowilYamlExtractor()
    return extractor(filename, options)
{'fraction_non_alphanumeric': 0.11104387291981846, 'fraction_numerical': 0.004841149773071104, 'mean_word_length': 1.877284595300261, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2863700', 'n_tokens_mistral': 965, 'n_tokens_neox': 907, 'n_words': 211}