<commit_before>
from typing import Tuple, List
from collections import defaultdict
from ..tree import Tree
from ..cat import Category
from ..combinator import (guess_combinator_by_triplet,
                          ja_default_binary_rules,
                          en_default_binary_rules,
                          unknown_combinator)
from ..tokens import Token
from lxml import etree
import logging

logger = logging.getLogger(__name__)

UNK_COMBINATOR = unknown_combinator()
BINARY_RULES = defaultdict(
    lambda: [UNK_COMBINATOR],
    {
        'en': en_default_binary_rules,
        'ja': ja_default_binary_rules
    }
)


def read_auto(filename, lang='en'):
    for line in open(filename):
        line = line.strip()
        if len(line) == 0:
            continue
        if line.startswith("ID"):
            name = line
        else:
            tree, tokens = Tree.of_auto(line, lang)
            yield name, tokens, tree


def read_xml(filename, lang='en'):
    binary_rules = BINARY_RULES[lang]
    def parse(tree):
        def rec(node):
            attrib = node.attrib
            if node.tag == 'rule':
                cat = Category.parse(attrib['cat'])
                children = [rec(child) for child in node.getchildren()]
                if len(children) == 1:
                    return Tree.make_unary(cat, children[0], lang)
                else:
                    assert len(children) == 2
                    left, right = children
                    combinator = guess_combinator_by_triplet(
                                    binary_rules, cat, left.cat, right.cat)
                    combinator = combinator or UNK_COMBINATOR
                    return Tree.make_binary(
                        cat, True, left, right, combinator, lang)
            else:
                assert node.tag == 'lf'
                cat = Category.parse(attrib['cat'])
                word = attrib['word']
                position = len(tokens)
                token = Token(word=attrib['word'],
                              pos=attrib['pos'],
                              entity=attrib['entity'],
                              lemma=attrib['lemma'],
                              chunk=attrib['chunk'])
                tokens.append(token)
                return Tree.make_terminal(word, cat, position, lang)
        tokens = []
        tree = rec(tree)
        return tree, tokens

    trees = etree.parse(filename).getroot().xpath('ccg')
    for tree in trees:
        name = '_'.join(f'{k}={v}' for k, v in tree.items())
        yield (name,) + parse(tree[0])


def read_jigg_xml(filename, lang='en'):
    binary_rules = BINARY_RULES[lang]
    # TODO
    def try_get_surface(token):
        if 'word' in token:
            return token.word
        elif 'surf' in token:
            return token.surf
        else:
            raise RuntimeError(
                'the attribute for the token\'s surface form is unknown')

    def parse(tree, tokens):
        counter = 0
        def rec(node):<commit_msg>fix a bug in a reader<commit_after>
from typing import Tuple, List
from collections import defaultdict
from ..tree import Tree
from ..cat import Category
from ..combinator import (guess_combinator_by_triplet,
                          ja_default_binary_rules,
                          en_default_binary_rules,
                          unknown_combinator)
from ..tokens import Token
from lxml import etree
import logging

logger = logging.getLogger(__name__)

UNK_COMBINATOR = unknown_combinator()
BINARY_RULES = defaultdict(
    lambda: [UNK_COMBINATOR],
    {
        'en': en_default_binary_rules,
        'ja': ja_default_binary_rules
    }
)


def read_auto(filename, lang='en'):
    for line in open(filename):
        line = line.strip()
        if len(line) == 0:
            continue
        if line.startswith("ID"):
            name = line
        else:
            tree, tokens = Tree.of_auto(line, lang)
            yield name, tokens, tree


def read_xml(filename, lang='en'):
    binary_rules = BINARY_RULES[lang]
    def parse(tree):
        def rec(node):
            attrib = node.attrib
            if node.tag == 'rule':
                cat = Category.parse(attrib['cat'])
                children = [rec(child) for child in node.getchildren()]
                if len(children) == 1:
                    return Tree.make_unary(cat, children[0], lang)
                else:
                    assert len(children) == 2
                    left, right = children
                    combinator = guess_combinator_by_triplet(
                                    binary_rules, cat, left.cat, right.cat)
                    combinator = combinator or UNK_COMBINATOR
                    return Tree.make_binary(
                        cat, True, left, right, combinator, lang)
            else:
                assert node.tag == 'lf'
                cat = Category.parse(attrib['cat'])
                word = attrib['word']
                position = len(tokens)
                token = Token(word=attrib['word'],
                              pos=attrib['pos'],
                              entity=attrib['entity'],
                              lemma=attrib['lemma'],
                              chunk=attrib['chunk'])
                tokens.append(token)
                return Tree.make_terminal(word, cat, position, lang)
        tokens = []
        tree = rec(tree)
        return tokens, tree

    trees = etree.parse(filename).getroot().xpath('ccg')
    for tree in trees:
        name = '_'.join(f'{k}={v}' for k, v in tree.items())
        yield (name,) + parse(tree[0])


def read_jigg_xml(filename, lang='en'):
    binary_rules = BINARY_RULES[lang]
    # TODO
    def try_get_surface(token):
        if 'word' in token:
            return token.word
        elif 'surf' in token:
            return token.surf
        else:
            raise RuntimeError(
                'the attribute for the token\'s surface form is unknown')

    def parse(tree, tokens):
        counter = 0
        def rec(node):
{'fraction_non_alphanumeric': 0.0976575387660838, 'fraction_numerical': 0.001979544704717915, 'mean_word_length': 1.6088640275387263, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2721100', 'n_tokens_mistral': 1698, 'n_tokens_neox': 1619, 'n_words': 424}