<commit_before>
from slimit import ast
from lxml.builder import E
import lxml.etree as ET

class XmlVisitor(object):

    def _make_indent(self):
        return ' '

    def visit(self, node):
        method = 'visit_%s' % node.__class__.__name__
        #print method, node,
        #try:
            #print node.value
        #except:
            #print
        return getattr(self, method, self.generic_visit)(node)

    def generic_visit(self, node):
        return 'GEN: %r' % node

    def visit_Program(self, node):
        program = E.program()
        for child in node:
            program.append(self.visit(child))
        return program

    def visit_Block(self, node):
        block = E.block()
        for child in node:
            #print child
            block.append(self.visit(child))
        return block

    def visit_VarStatement(self, node):
        var = E.var()
        for child in node:
            var.append(self.visit(child))
        return var

    def visit_VarDecl(self, node):
        var_decl = E.var_decl()
        var_decl.append(E.identifider(self.visit(node.identifier)))
        if node.initializer is not None:
            var_decl.append(E.initializer(self.visit(node.initializer)))
        return var_decl

    def visit_Identifier(self, node):
        if isinstance(node.value, (int, float)):
            return E.identifier(node.value)
        elif isinstance(node.value, (str, unicode)):
            return E.identifier(node.value)

    def visit_Assign(self, node):
        if node.op == ':':
            template = '%s%s %s'
        else:
            template = '%s %s %s'
        if getattr(node, '_parens', False):
            template = '(%s)' % template
        #print node.__dict__
        assign = E.assign()
        assign.append(E.left(self.visit(node.left)))
        assign.append(E.operator(self.visit_Operator(node.op)))
        assign.append(E.right(self.visit(node.right)))
        return assign

    def visit_GetPropAssign(self, node):
        get = E.get()
        get.append(E.property(self.visit(node.prop_name)))
        body = E.body()
        for el in node.elements:
            body.append(self.visit(el))
        get.append(body)
        return get

    def visit_SetPropAssign(self, node):
        template = 'set %s(%s) {\n%s\n%s}'
        if getattr(node, '_parens', False):
            template = '(%s)' % template
        if len(node.parameters) > 1:
            raise SyntaxError(
                'Setter functions must have one argument: %s' % node)
        params = ','.join(self.visit(param) for param in node.parameters)
        body = '\n'.join(
            (self._make_indent() + self.visit(el))
            for el in node.elements
            )
        tail = self._make_indent()
        return template % (self.visit(node.prop_name), params, body, tail)

    def visit_Number(self, node):
        return E.number(node.value)

    def visit_Comma(self, node):
        comma = E.comma(E.left(self.visit(node.left)),
                        E.right(self.visit(node.right)))
        return comma

    def visit_EmptyStatement(self, node):
        return E.empty(node.value)

    def visit_If(self, node):
        ifel = ET.Element("if")
        if node.predicate is not None:
            ifel.append(E.predicate(self.visit(node.predicate)))
        ifel.append(E.then(self.visit(node.consequent)))
        if node.alternative is not None:
            ifel.append(ET.Element("else", self.visit(node.alternative)))
        return ifel

    def visit_Boolean(self, node):
        return E.boolean(node.value)

    def visit_For(self, node):
        forel = ET.Element("for")
        if node.init is not None:
            forel.append(E.init(self.visit(node.init)))
        if node.init is None:
            forel.append(E.init())
        elif isinstance(node.init, (ast.Assign, ast.Comma, ast.FunctionCall,
                                    ast.UnaryOp, ast.Identifier, ast.BinOp,
                                    ast.Conditional, ast.Regex, ast.NewExpr)):
            pass
        else:
            pass
        if node.cond is not None:
            forel.append(E.condition(self.visit(node.cond)))
        if node.count is not None:
            forel.append(E.post(self.visit(node.count)))
        forel.append(E.statement(self.visit(node.statement)))
        return forel

    def visit_ForIn(self, node):
        if isinstance(node.item, ast.VarDecl):
            template = 'for (var %s in %s) '
        else:
            template = 'for (%s in %s) '
        forel = ET.Element("forin")
        forel.append(E.variable(self.visit(node.item)))
        forel.append(E.object(self.visit(node.iterable)))
        return forel

    def visit_BinOp(self, node):
        binop = E.binaryoperation()
        binop.append(E.left(self.visit(node.left)))
        binop.append(E.operator(self.visit(node.op)))
        binop.append(E.right(self.visit(node.right)))
        return binop

        #if getattr(node, '_parens', False):
            #template = '(%s %s %s)'
        #else:
            #template = '%s %s %s'
        #return template % (
            #self.visit(node.left), node.op, self.visit(node.right))

    def visit_UnaryOp(self, node):
        unary = E.unaryoperation()
        if node.postfix:
            unary.append(self.visit(node.value))
            unary.append(E.operator(self.visit(node.op)))
        elif node.op in ('delete', 'void', 'typeof'):
            #s = '%s %s' % (node.op, s)
            unary.append(E.operation(node.op))
            unary.append(self.visit(node.value))
        else:
            unary.append(E.operation(node.op))
            unary.append(self.visit(node.value))
        return unary

    def visit_ExprStatement(self, node):
        return self.visit(node.expr)

    def visit_DoWhile(self, node):
        dowhile = E.dowhile()
        dowhile.append(E.statement(self.visit(node.statement)))
        dowhile.append(ET.Element("while", self.visit(node.predicate)))
        return dowhile

    def visit_While(self, node):
        whileel = ET.Element("while")
        whileel.append(E.predicate(self.visit(node.predicate)))
        whileel.append(E.statement(self.visit(node.statement)))<commit_msg>Convert simple unary operation on numbers<commit_after>
from slimit import ast
from lxml.builder import E
import lxml.etree as ET

class XmlVisitor(object):

    def _make_indent(self):
        return ' '

    def visit(self, node):
        method = 'visit_%s' % node.__class__.__name__
        #print method, node,
        #try:
            #print node.value
        #except:
            #print
        return getattr(self, method, self.generic_visit)(node)

    def generic_visit(self, node):
        return 'GEN: %r' % node

    def visit_Program(self, node):
        program = E.program()
        for child in node:
            program.append(self.visit(child))
        return program

    def visit_Block(self, node):
        block = E.block()
        for child in node:
            #print child
            block.append(self.visit(child))
        return block

    def visit_VarStatement(self, node):
        var = E.var()
        for child in node:
            var.append(self.visit(child))
        return var

    def visit_VarDecl(self, node):
        var_decl = E.var_decl()
        var_decl.append(self.visit(node.identifier))
        if node.initializer is not None:
            var_decl.append(E.initializer(self.visit(node.initializer)))
        return var_decl

    def visit_Identifier(self, node):
        if isinstance(node.value, (int, float)):
            return E.identifier(node.value)
        elif isinstance(node.value, (str, unicode)):
            return E.identifier(node.value)

    def visit_Assign(self, node):
        if node.op == ':':
            template = '%s%s %s'
        else:
            template = '%s %s %s'
        if getattr(node, '_parens', False):
            template = '(%s)' % template
        #print node.__dict__
        assign = E.assign()
        assign.append(E.left(self.visit(node.left)))
        assign.append(E.operator(self.visit_Operator(node.op)))
        assign.append(E.right(self.visit(node.right)))
        return assign

    def visit_GetPropAssign(self, node):
        get = E.get()
        get.append(E.property(self.visit(node.prop_name)))
        body = E.body()
        for el in node.elements:
            body.append(self.visit(el))
        get.append(body)
        return get

    def visit_SetPropAssign(self, node):
        template = 'set %s(%s) {\n%s\n%s}'
        if getattr(node, '_parens', False):
            template = '(%s)' % template
        if len(node.parameters) > 1:
            raise SyntaxError(
                'Setter functions must have one argument: %s' % node)
        params = ','.join(self.visit(param) for param in node.parameters)
        body = '\n'.join(
            (self._make_indent() + self.visit(el))
            for el in node.elements
            )
        tail = self._make_indent()
        return template % (self.visit(node.prop_name), params, body, tail)

    def visit_Number(self, node):
        return E.number(node.value)

    def visit_Comma(self, node):
        comma = E.comma(E.left(self.visit(node.left)),
                        E.right(self.visit(node.right)))
        return comma

    def visit_EmptyStatement(self, node):
        return E.empty(node.value)

    def visit_If(self, node):
        ifel = ET.Element("if")
        if node.predicate is not None:
            ifel.append(E.predicate(self.visit(node.predicate)))
        ifel.append(E.then(self.visit(node.consequent)))
        if node.alternative is not None:
            ifel.append(ET.Element("else", self.visit(node.alternative)))
        return ifel

    def visit_Boolean(self, node):
        return E.boolean(node.value)

    def visit_For(self, node):
        forel = ET.Element("for")
        if node.init is not None:
            forel.append(E.init(self.visit(node.init)))
        if node.init is None:
            forel.append(E.init())
        elif isinstance(node.init, (ast.Assign, ast.Comma, ast.FunctionCall,
                                    ast.UnaryOp, ast.Identifier, ast.BinOp,
                                    ast.Conditional, ast.Regex, ast.NewExpr)):
            pass
        else:
            pass
        if node.cond is not None:
            forel.append(E.condition(self.visit(node.cond)))
        if node.count is not None:
            forel.append(E.post(self.visit(node.count)))
        forel.append(E.statement(self.visit(node.statement)))
        return forel

    def visit_ForIn(self, node):
        if isinstance(node.item, ast.VarDecl):
            template = 'for (var %s in %s) '
        else:
            template = 'for (%s in %s) '
        forel = ET.Element("forin")
        forel.append(E.variable(self.visit(node.item)))
        forel.append(E.object(self.visit(node.iterable)))
        return forel

    def visit_BinOp(self, node):
        binop = E.binaryoperation()
        binop.append(E.left(self.visit(node.left)))
        binop.append(E.operator(self.visit(node.op)))
        binop.append(E.right(self.visit(node.right)))
        return binop

        #if getattr(node, '_parens', False):
            #template = '(%s %s %s)'
        #else:
            #template = '%s %s %s'
        #return template % (
            #self.visit(node.left), node.op, self.visit(node.right))

    def visit_UnaryOp(self, node):
        unary = E.unaryoperation()
        if node.postfix:
            unary.append(self.visit(node.value))
            unary.append(E.operator(self.visit(node.op)))
        elif node.op in ('delete', 'void', 'typeof'):
            #s = '%s %s' % (node.op, s)
            unary.append(E.operation(node.op))
            unary.append(self.visit(node.value))
        else:
            # convert things like "+3.14" and "-22"
            if node.op in ("-", "+") and isinstance(node.value, ast.Number):
                node.value.value = "%s%s" % (node.op, node.value.value)
                return self.visit(node.value)
            else:
                unary.append(E.operation(node.op))
                unary.append(self.visit(node.value))
        return unary

    def visit_ExprStatement(self, node):
        return self.visit(node.expr)

    def visit_DoWhile(self, node):
        dowhile = E.dowhile()
        dowhile.append(E.statement(self.visit(node.statement)))
        dowhile.append(ET.Element("while", self.visit(node.predicate)))
        return dowhile

    def visit_While(self, node):
        whileel = ET.Element("while")
        whileel.append(E.predicate(self.visit(node.predicate)))
        whileel.append(E.statement(self.visit(node.statement)))
{'fraction_non_alphanumeric': 0.1298416183158225, 'fraction_numerical': 0.0005488474204171241, 'mean_word_length': 2.0268153773137163, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7618594', 'n_tokens_mistral': 3939, 'n_tokens_neox': 3868, 'n_words': 811}