<commit_before>import codepy.cgen
from codepy.cgen import *
import xml.etree.ElementTree as ElementTree

# these are additional classes that, along with codepy's classes, let
# programmers express the C code as a real AST (not the hybrid AST/strings/etc
# that codepy implements.

#TODO: add all of CodePy's classes we want to support

class CNumber(Generable):
    def __init__(self, num):
        self.num = num
        self._fields = []

#    def __str__(self):
#        return str(self.num)

    def to_xml(self):
        return ElementTree.Element("CNumber", attrib={"num":str(self.num)})

    def generate(self, with_semicolon=False):
        if with_semicolon:
            # This node type does not represent a complete C++ statement
            raise ValueError
        yield str(self.num)

class String(Generable):
    def __init__(self, text):
        self.text = text

    def generate(self):
        yield '\"%s\"' % self.text

class CName(Generable):
    def __init__(self, name):
        self.name = name
        self._fields = []

    def to_xml(self):
        return ElementTree.Element("CName", attrib={"name":str(self.name)})

    def generate(self, with_semicolon=False):
        if with_semicolon:
            # This node type does not represent a complete C++ statement
            raise ValueError
        yield self.name

class Expression(Generable):
    def __init__(self):
        super(Expression, self).__init__()
        self._fields = []

    def generate(self, with_semicolon=False):
        yield ""
        

class BinOp(Expression):
    def __init__(self, left, op, right):
        self.left = left
        self.op = op
        self.right = right
        self._fields = ['left', 'right']

    def generate(self, with_semicolon=False):
        yield "(%s %s %s)" % (self.left, self.op, self.right) + (";" if with_semicolon else "")

    def to_xml(self):
        node = ElementTree.Element("BinOp", attrib={"op":str(self.op)})
        left = ElementTree.SubElement(node, "left")
        left.append(self.left.to_xml())
        right = ElementTree.SubElement(node, "right")
        right.append(self.right.to_xml())
        return node

class UnaryOp(Expression):
    def __init__(self, op, operand):
        self.op = op
        self.operand = operand
        self._fields = ['operand']

    def generate(self, with_semicolon=False):
        yield "(%s(%s))" % (self.op, self.operand) + (";" if with_semicolon else "")

    def to_xml(self):
        node = ElementTree.Element("UnaryOp", attrib={"op":str(self.op)})
        operand = ElementTree.SubElement(node, "operand")
        operand.append(self.operand.to_xml())
        return node

class Subscript(Expression):
    def __init__(self, value, index):
        self.value = value
        self.index = index
        self._fields = ['value', 'index']

    def generate(self, with_semicolon=False):
        yield "%s[%s]" % (self.value, self.index)

    def to_xml(self):
        node = ElementTree.Element("Subscript")
        ElementTree.SubElement(node, "value").append(self.value.to_xml())
        ElementTree.SubElement(node, "index").append(self.index.to_xml())
        return node

class Call(Expression):
    def __init__(self, func, args):
        self.func = func
        self.args = args
        self._fields = ['func', 'args']

    def generate(self, with_semicolon=False): 
        yield "%s(%s)" % (self.func, ", ".join(map(str, self.args))) + (";" if with_semicolon else "")

    def to_xml(self):
        node = ElementTree.Element("Call", attrib={"func":str(self.func)})
        args = ElementTree.SubElement(node, "args")
        for x in self.args:
            args.append(x.to_xml())
        return node


class PostfixUnaryOp(Expression):
    def __init__(self, operand, op):
        self.operand = operand
        self.op = op
        self._fields = ['op', 'operand']

    def generate(self, with_semicolon=False):
        yield "((%s)%s)" % (self.operand, self.op) + (";" if with_semicolon else "")

    def to_xml(self):
        node = ElementTree.Element("PostfixUnaryOp", attrib={"op":str(self.op)})
        operand = ElementTree.SubElement(node, "operand")
        operand.append(self.operand.to_xml())
        return node


class ConditionalExpr(Expression):
    def __init__(self, test, body, orelse):
        self.test = test
        self.body = body
        self.orelse = orelse
        self._fields = ['test', 'body', 'orelse']

    def generate(self, with_semicolon=False):
        yield "(%s ? %s : %s)" % (self.test, self.body, self.orelse) + (";" if with_semicolon else "")

    def to_xml(self):
        node = ElementTree.Element("ConditionalExpr")
        ElementTree.SubElement(node, "test").append(self.test.to_xml())
        ElementTree.SubElement(node, "body").append(self.body.to_xml())
        ElementTree.SubElement(node, "orelse").append(self.orelse.to_xml())
        return node

class TypeCast(Expression):
    # "type" should be a declaration with an empty variable name
    # e.g. TypeCast(Pointer(Value('int', '')), ...)

    def __init__(self, tp, value):
        self.tp = tp
        self.value = value
        self._fields = ['tp', 'value']

    def generate(self):
        yield "((%s)%s)" % (self.tp.inline(), self.value)

#class ForInitializer(codepy.cgen.Initializer):
#    def __str__(self):
#        return super(ForInitializer, self).__str__()[0:-1]

class Initializer(codepy.cgen.Initializer):
    def generate(self, with_semicolon=False):
        tp_lines, tp_decl = self.vdecl.get_decl_pair()
        tp_lines = list(tp_lines)
        for line in tp_lines[:-1]:
            yield line
        yield "%s %s = %s" % (tp_lines[-1], tp_decl, self.data) + (";" if with_semicolon else "")
    

class RawFor(codepy.cgen.For):
    def __init__(self, start, condition, update, body):
        super(RawFor, self).__init__(start, condition, update, body)
        self._fields = ['start', 'condition', 'update', 'body']

    def to_xml(self):
        node = ElementTree.Element("For")
        if (not isinstance(self.start, str)):
            ElementTree.SubElement(node, "start").append(self.start.to_xml())
        else:
            ElementTree.SubElement(node,"start").text = self.start
            
        if (not isinstance(self.condition, str)):
            ElementTree.SubElement(node, "condition").append(self.condition.to_xml())
        else:
            ElementTree.SubElement(node, "condition").text = self.condition<commit_msg>small bugfix for semicolon handling.<commit_after>import codepy.cgen
from codepy.cgen import *
import xml.etree.ElementTree as ElementTree

# these are additional classes that, along with codepy's classes, let
# programmers express the C code as a real AST (not the hybrid AST/strings/etc
# that codepy implements.

#TODO: add all of CodePy's classes we want to support

class CNumber(Generable):
    def __init__(self, num):
        self.num = num
        self._fields = []

#    def __str__(self):
#        return str(self.num)

    def to_xml(self):
        return ElementTree.Element("CNumber", attrib={"num":str(self.num)})

    def generate(self, with_semicolon=False):
        if with_semicolon:
            # This node type does not represent a complete C++ statement
            raise ValueError
        yield str(self.num)

class String(Generable):
    def __init__(self, text):
        self.text = text

    def generate(self):
        yield '\"%s\"' % self.text

class CName(Generable):
    def __init__(self, name):
        self.name = name
        self._fields = []

    def to_xml(self):
        return ElementTree.Element("CName", attrib={"name":str(self.name)})

    def generate(self, with_semicolon=False):
        if with_semicolon:
            # This node type does not represent a complete C++ statement
            raise ValueError
        yield self.name

class Expression(Generable):
    def __init__(self):
        super(Expression, self).__init__()
        self._fields = []

    def generate(self, with_semicolon=False):
        yield ""
        

class BinOp(Expression):
    def __init__(self, left, op, right):
        self.left = left
        self.op = op
        self.right = right
        self._fields = ['left', 'right']

    def generate(self, with_semicolon=False):
        yield "(%s %s %s)" % (self.left, self.op, self.right) + (";" if with_semicolon else "")

    def to_xml(self):
        node = ElementTree.Element("BinOp", attrib={"op":str(self.op)})
        left = ElementTree.SubElement(node, "left")
        left.append(self.left.to_xml())
        right = ElementTree.SubElement(node, "right")
        right.append(self.right.to_xml())
        return node

class UnaryOp(Expression):
    def __init__(self, op, operand):
        self.op = op
        self.operand = operand
        self._fields = ['operand']

    def generate(self, with_semicolon=False):
        yield "(%s(%s))" % (self.op, self.operand) + (";" if with_semicolon else "")

    def to_xml(self):
        node = ElementTree.Element("UnaryOp", attrib={"op":str(self.op)})
        operand = ElementTree.SubElement(node, "operand")
        operand.append(self.operand.to_xml())
        return node

class Subscript(Expression):
    def __init__(self, value, index):
        self.value = value
        self.index = index
        self._fields = ['value', 'index']

    def generate(self, with_semicolon=False):
        yield "%s[%s]" % (self.value, self.index)

    def to_xml(self):
        node = ElementTree.Element("Subscript")
        ElementTree.SubElement(node, "value").append(self.value.to_xml())
        ElementTree.SubElement(node, "index").append(self.index.to_xml())
        return node

class Call(Expression):
    def __init__(self, func, args):
        self.func = func
        self.args = args
        self._fields = ['func', 'args']

    def generate(self, with_semicolon=False): 
        yield "%s(%s)" % (self.func, ", ".join(map(str, self.args))) + (";" if with_semicolon else "")

    def to_xml(self):
        node = ElementTree.Element("Call", attrib={"func":str(self.func)})
        args = ElementTree.SubElement(node, "args")
        for x in self.args:
            args.append(x.to_xml())
        return node


class PostfixUnaryOp(Expression):
    def __init__(self, operand, op):
        self.operand = operand
        self.op = op
        self._fields = ['op', 'operand']

    def generate(self, with_semicolon=False):
        yield "((%s)%s)" % (self.operand, self.op) + (";" if with_semicolon else "")

    def to_xml(self):
        node = ElementTree.Element("PostfixUnaryOp", attrib={"op":str(self.op)})
        operand = ElementTree.SubElement(node, "operand")
        operand.append(self.operand.to_xml())
        return node


class ConditionalExpr(Expression):
    def __init__(self, test, body, orelse):
        self.test = test
        self.body = body
        self.orelse = orelse
        self._fields = ['test', 'body', 'orelse']

    def generate(self, with_semicolon=False):
        yield "(%s ? %s : %s)" % (self.test, self.body, self.orelse) + (";" if with_semicolon else "")

    def to_xml(self):
        node = ElementTree.Element("ConditionalExpr")
        ElementTree.SubElement(node, "test").append(self.test.to_xml())
        ElementTree.SubElement(node, "body").append(self.body.to_xml())
        ElementTree.SubElement(node, "orelse").append(self.orelse.to_xml())
        return node

class TypeCast(Expression):
    # "type" should be a declaration with an empty variable name
    # e.g. TypeCast(Pointer(Value('int', '')), ...)

    def __init__(self, tp, value):
        self.tp = tp
        self.value = value
        self._fields = ['tp', 'value']

    def generate(self, with_semicolon=False):
        yield "((%s)%s)" % (self.tp.inline(), self.value)

#class ForInitializer(codepy.cgen.Initializer):
#    def __str__(self):
#        return super(ForInitializer, self).__str__()[0:-1]

class Initializer(codepy.cgen.Initializer):
    def generate(self, with_semicolon=False):
        tp_lines, tp_decl = self.vdecl.get_decl_pair()
        tp_lines = list(tp_lines)
        for line in tp_lines[:-1]:
            yield line
        yield "%s %s = %s" % (tp_lines[-1], tp_decl, self.data) + (";" if with_semicolon else "")
    

class RawFor(codepy.cgen.For):
    def __init__(self, start, condition, update, body):
        super(RawFor, self).__init__(start, condition, update, body)
        self._fields = ['start', 'condition', 'update', 'body']

    def to_xml(self):
        node = ElementTree.Element("For")
        if (not isinstance(self.start, str)):
            ElementTree.SubElement(node, "start").append(self.start.to_xml())
        else:
            ElementTree.SubElement(node,"start").text = self.start
            
        if (not isinstance(self.condition, str)):
            ElementTree.SubElement(node, "condition").append(self.condition.to_xml())
        else:
            ElementTree.SubElement(node, "condition").text = self.condition
{'fraction_non_alphanumeric': 0.141296823321283, 'fraction_numerical': 0.0006153372817475579, 'mean_word_length': 2.769788344447666, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 44}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2093368', 'n_tokens_mistral': 4102, 'n_tokens_neox': 4061, 'n_words': 930}