<commit_before>"""
Module to transform an AST (Abstract Syntax Tree) to a 
Python dict structure
"""
from collections import defaultdict

from plyplus import STransformer, is_stree

from tokens import ATTRIBUTE_NAMES, COMPOSITE_NAMES

def plural(s):
    if s == 'points':
        return s
    elif s.endswith('s'):
        return s+'es'
    return s+'s'

class MapFile2Dict__Transformer(STransformer):
    def start(self, t):
        t ,= t.tail
        assert t[0] == 'composite'
        assert t[1].lower() == 'map'
        return t[2]

    def composite(self, t):
        if len(t.tail) == 3:
            # Parser artifact. See LINE-BREAK FLUIDITY in parsing_decisions.txt
            type_, attr, body = t.tail
        else:
            type_, body = t.tail
            attr = None
        if isinstance(body, tuple):
            assert body[0] == 'attr' or body[1] == 'points', body  # Parser artifcats
            body = [body]
        else:
            body = body.tail
        type_ = type_.tail[0].lower()
        assert type_ in COMPOSITE_NAMES

        if attr:
            body = [attr] + body

        for x in body:
            assert isinstance(x, tuple), x

        composites = defaultdict(list)

        d = {}
        for itemtype, k, v in body:
            if itemtype == 'attr':
                d[k] = v
            elif itemtype == 'composite':
                composites[k].append(v)
            else:
                assert False, item

        for k, v in composites.items():
            d[plural(k)] = v

        d['__type__'] = type_
        return ('composite', type_, d)

    def attr(self, t):
        name = t.tail[0].tail[0]
        if is_stree(name):
            name = name.tail[0] # Solve a parser artifact for composite names
        name = name.lower()
        assert name in ATTRIBUTE_NAMES, name
        value = t.tail[1:]
        if len(value) == 1:
            value ,= value
        return 'attr', name, value

    def projection(self, t):
        return ('composite', 'projection', t.tail)
    def metadata(self, t):
        return ('composite', 'metadata', t.tail)<commit_msg>Spelling and add hack for PROCESSING directive<commit_after>"""
Module to transform an AST (Abstract Syntax Tree) to a 
Python dict structure
"""

from collections import defaultdict

from plyplus import STransformer, is_stree

from tokens import ATTRIBUTE_NAMES, COMPOSITE_NAMES

def plural(s):
    if s == 'points':
        return s
    elif s.endswith('s'):
        return s+'es'
    return s+'s'

class MapFile2Dict__Transformer(STransformer):
    def start(self, t):
        t ,= t.tail
        assert t[0] == 'composite'
        assert t[1].lower() == 'map'
        return t[2]

    def composite(self, t):
        if len(t.tail) == 3:
            # Parser artefact. See LINE-BREAK FLUIDITY in parsing_decisions.txt
            type_, attr, body = t.tail
        else:
            type_, body = t.tail
            attr = None
        if isinstance(body, tuple):
            assert body[0] == 'attr' or body[1] == 'points', body  # Parser artefacts
            body = [body]
        else:
            body = body.tail
        type_ = type_.tail[0].lower()
        assert type_ in COMPOSITE_NAMES

        if attr:
            body = [attr] + body

        for x in body:
            assert isinstance(x, tuple), x

        composites = defaultdict(list)

        d = {}
        for itemtype, k, v in body:             
            if itemtype == 'attr':

                if k == 'processing':
                    # HACK as PROCESSING can be repeated
                    if 'processing' not in d.keys():
                        d[k] = [v]
                    else:
                        d[k].append(v)
                else:
                    d[k] = v
            elif itemtype == 'composite':
                composites[k].append(v)
            else:
                assert False, item

        for k, v in composites.items():
            d[plural(k)] = v

        d['__type__'] = type_
        return ('composite', type_, d)

    def attr(self, t):
        name = t.tail[0].tail[0]
        if is_stree(name):
            name = name.tail[0] # Solve a parser artefact for composite names
        name = name.lower()
        assert name in ATTRIBUTE_NAMES, name
        value = t.tail[1:]
        if len(value) == 1:
            value ,= value
        return 'attr', name, value

    def projection(self, t):
        return ('composite', 'projection', t.tail)
    def metadata(self, t):
        return ('composite', 'metadata', t.tail)
{'fraction_non_alphanumeric': 0.11047786058746165, 'fraction_numerical': 0.005699254712845244, 'mean_word_length': 1.6746776084407973, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4117087', 'n_tokens_mistral': 1402, 'n_tokens_neox': 1345, 'n_words': 420}