<commit_before>import json
import sys
import logging
import re

from collections import defaultdict, OrderedDict
from typing import NamedTuple, Union, List
from pathlib import Path


logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
                    level=logging.INFO)
logger = logging.getLogger(__name__)


test = """
(TOP (Sm."L" (Sm."L" (CPt."L" (IP-MS."L" (IP-MS."L" (IP-MS."L" (IP-MS *pro*) (<IP-MS\IP-MS> 「)) (<IP-MS\IP-MS>."R" (<<IP-MS\IP-MS>/<IP-MS\IP-MS>>."L" (<<IP-MS\IP-MS>/<IP-MS\IP-MS>>."L" (<<IP-MS\IP-MS>/<IP-MS\IP-MS>>."L" (PPs *pro*) (<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>."R" (<<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>/<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>>."L" (<<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>/<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>> さあ) (<<<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>/<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>>\<<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>/<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>>> 、)) (<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>."FCLeft1" (<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>."FCLeft1" (<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>> 流れ) (<<<IP-MS\IP-MS>/<IP-MS\IP-MS>>\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>> て)) (<<<IP-MS\IP-MS>/<IP-MS\IP-MS>>\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>> 来る)))) (<<<IP-MS\IP-MS>/<IP-MS\IP-MS>>\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>> ぞ)) (<<<IP-MS\IP-MS>/<IP-MS\IP-MS>>\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>> 。)) (<IP-MS\IP-MS>."FCLeft1" (<IP-MS\IP-MS>."L" (PPs みんな) (<PPs\<IP-MS\IP-MS>> とれ)) (<IP-MS\IP-MS> 。)))) (<IP-MS\IP-MS> 」)) (<IP-MS\CPt> と)) (<CPt\Sm>."L" (PPs."L" (NP 一郎) (<NP\PPs> が)) (<PPs\<CPt\Sm>>."FCLeft2" (<PPs\<CPt\Sm>>."FCLeft2" (<PPs\<CPt\Sm>> 言い) (<Sm\Sm> まし)) (<Sm\Sm> た)))) (<Sm\Sm> 。)) (ID 771_aozora_Miyazawa-1934;JP))
"""


Category = Union['Atomic', 'Functor']
Node = Union['Leaf', 'Tree']


class Atomic(NamedTuple):
    string: str

    def __str__(self):
        return self.string


class Functor(NamedTuple):
    left: str
    slash: str
    right: str

    def __str__(self):
        """
        convert to CCG version
        """
        def rec(cat):
            if isinstance(cat, Atomic):
                return str(cat)
            right = rec(cat.right if cat.slash == '\\' else cat.left)
            left = rec(cat.left if cat.slash == '\\' else cat.right)
            return f'({right}{self.slash}{left})'
        return rec(self)[1:-1]
<commit_msg>fix a bug in keyaki reader<commit_after>import json
import sys
import logging
import re

from collections import defaultdict, OrderedDict
from typing import NamedTuple, Union, List
from pathlib import Path


logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s',
                    level=logging.INFO)
logger = logging.getLogger(__name__)


test = """
(TOP (Sm."L" (Sm."L" (CPt."L" (IP-MS."L" (IP-MS."L" (IP-MS."L" (IP-MS *pro*) (<IP-MS\IP-MS> 「)) (<IP-MS\IP-MS>."R" (<<IP-MS\IP-MS>/<IP-MS\IP-MS>>."L" (<<IP-MS\IP-MS>/<IP-MS\IP-MS>>."L" (<<IP-MS\IP-MS>/<IP-MS\IP-MS>>."L" (PPs *pro*) (<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>."R" (<<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>/<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>>."L" (<<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>/<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>> さあ) (<<<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>/<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>>\<<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>/<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>>> 、)) (<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>."FCLeft1" (<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>>."FCLeft1" (<PPs\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>> 流れ) (<<<IP-MS\IP-MS>/<IP-MS\IP-MS>>\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>> て)) (<<<IP-MS\IP-MS>/<IP-MS\IP-MS>>\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>> 来る)))) (<<<IP-MS\IP-MS>/<IP-MS\IP-MS>>\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>> ぞ)) (<<<IP-MS\IP-MS>/<IP-MS\IP-MS>>\<<IP-MS\IP-MS>/<IP-MS\IP-MS>>> 。)) (<IP-MS\IP-MS>."FCLeft1" (<IP-MS\IP-MS>."L" (PPs みんな) (<PPs\<IP-MS\IP-MS>> とれ)) (<IP-MS\IP-MS> 。)))) (<IP-MS\IP-MS> 」)) (<IP-MS\CPt> と)) (<CPt\Sm>."L" (PPs."L" (NP 一郎) (<NP\PPs> が)) (<PPs\<CPt\Sm>>."FCLeft2" (<PPs\<CPt\Sm>>."FCLeft2" (<PPs\<CPt\Sm>> 言い) (<Sm\Sm> まし)) (<Sm\Sm> た)))) (<Sm\Sm> 。)) (ID 771_aozora_Miyazawa-1934;JP))
"""


Category = Union['Atomic', 'Functor']
Node = Union['Leaf', 'Tree']


class Atomic(NamedTuple):
    string: str

    def __str__(self):
        return self.string


class Functor(NamedTuple):
    left: str
    slash: str
    right: str

    def __str__(self):
        """
        convert to CCG version
        """
        def rec(cat):
            if isinstance(cat, Atomic):
                return str(cat)
            right = rec(cat.right if cat.slash == '\\' else cat.left)
            left = rec(cat.left if cat.slash == '\\' else cat.right)
            return f'({right}{cat.slash}{left})'
        return rec(self)[1:-1]

{'fraction_non_alphanumeric': 0.3128755364806867, 'fraction_numerical': 0.006008583690987125, 'mean_word_length': 2.903685092127303, 'pattern_counts': {'":': 0, '<': 223, '<?xml version=': 0, '>': 223, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4490226', 'n_tokens_mistral': 2331, 'n_tokens_neox': 2238, 'n_words': 299}