<commit_before>##
# Copyright (c) 2008-2010 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##


import ast
import copy
import itertools
import functools

from . import ast as py_ast
from semantix.exceptions import SemantixError
from semantix.utils.lang.generic import ast as generic_ast


_Token = object()


class AstToPyAstConverter:
    @classmethod
    def convert(cls, node:ast.AST):
        name = node.__class__.__name__

        new_cls = getattr(py_ast, 'Py' + name, None)

        if new_cls is None:
            raise SemantixError('unknown python ast class "%s"' % name)

        new_node = new_cls()

        for field, value in py_ast.iter_fields(node):
            if isinstance(value, list):
                new_value = []
                for item in value:
                    if isinstance(item, ast.AST):
                        new_item = cls.convert(item)
                        new_item.parent = new_node
                        new_value.append(new_item)
                    else:
                        new_value.append(item)
                setattr(new_node, field, new_value)
            elif isinstance(value, ast.AST):
                new_item = cls.convert(value)
                new_item.parent = new_node
                setattr(new_node, field, new_item)
            else:
                setattr(new_node, field, value)

        return new_node


class PyAstToGeneric:
    CTX_MAP = {
        py_ast.PyLoad:  generic_ast.NAME_CONTEXT_LOAD,
        py_ast.PyStore: generic_ast.NAME_CONTEXT_STORE,
        py_ast.PyDel:   generic_ast.NAME_CONTEXT_DELETE
    }

    BOOLOP_MAP = {
        py_ast.PyAnd:       generic_ast.And,
        py_ast.PyOr:        generic_ast.Or
    }
<commit_msg>utils.lang.python.convert: Copy 'lineno' and 'col_offset' properties<commit_after>##
# Copyright (c) 2008-2010 Sprymix Inc.
# All rights reserved.
#
# See LICENSE for details.
##


import ast
import copy
import itertools
import functools

from . import ast as py_ast
from semantix.exceptions import SemantixError
from semantix.utils.lang.generic import ast as generic_ast


_Token = object()


class AstToPyAstConverter:
    @classmethod
    def convert(cls, node:ast.AST):
        name = node.__class__.__name__

        new_cls = getattr(py_ast, 'Py' + name, None)

        if new_cls is None:
            raise SemantixError('unknown python ast class "%s"' % name)

        new_node = new_cls()

        for field, value in py_ast.iter_fields(node):
            if isinstance(value, list):
                new_value = []
                for item in value:
                    if isinstance(item, ast.AST):
                        new_item = cls.convert(item)
                        if hasattr(item, 'lineno'):
                            new_item.lineno = item.lineno
                            new_item.col_offset = item.col_offset
                        new_item.parent = new_node
                        new_value.append(new_item)
                    else:
                        new_value.append(item)
                setattr(new_node, field, new_value)
            elif isinstance(value, ast.AST):
                new_item = cls.convert(value)
                new_item.parent = new_node
                if hasattr(value, 'lineno'):
                    new_item.lineno = value.lineno
                    new_item.col_offset = value.col_offset
                setattr(new_node, field, new_item)
            else:
                setattr(new_node, field, value)

        return new_node


class PyAstToGeneric:
    CTX_MAP = {
        py_ast.PyLoad:  generic_ast.NAME_CONTEXT_LOAD,
        py_ast.PyStore: generic_ast.NAME_CONTEXT_STORE,
        py_ast.PyDel:   generic_ast.NAME_CONTEXT_DELETE
    }

    BOOLOP_MAP = {
        py_ast.PyAnd:       generic_ast.And,
        py_ast.PyOr:        generic_ast.Or
    }

{'fraction_non_alphanumeric': 0.07875647668393783, 'fraction_numerical': 0.004145077720207254, 'mean_word_length': 1.7598284488920657, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5438464', 'n_tokens_mistral': 1190, 'n_tokens_neox': 1125, 'n_words': 259}