<commit_before>import os
import model

class PythonSerializer(object):

    libname = "prophy"
    primitive_types = set(x + y for x in "uir" for y in ["8", "16", "32", "64"])

    def __init__(self, output_dir = "."):
        self.output_dir = output_dir

    def serialize_string(self, nodes, header = True):
        out = ""
        if header:
            out += self.__render_header()
            if nodes:
                out += "\n"
        out += "\n".join(self.__render(node) for node in nodes)
        return out

    def serialize(self, nodes, basename):
        path = os.path.join(self.output_dir, basename + ".py")
        out = self.serialize_string(nodes)
        open(path, "w").write(out)

    def __render_header(self):
        return """\
import %s

def bitMaskOr(x, y):
    return x | y

def shiftLeft(x, y):
    return x << y
""" % self.libname

    def __render_include(self, include):
        return "from %s import *\n" % include

    def __render_constant(self, constant):
        return "%s = %s\n" % constant

    def __render_typedef(self, typedef):
        return "%s = %s\n" % (typedef.name, ".".join((self.libname, typedef.type))
                                            if typedef.type in self.primitive_types
                                            else typedef.type)

    def __render_enum_members(self, members):<commit_msg>easy way to express rendering<commit_after>import os
import model

class PythonSerializer(object):

    libname = "prophy"
    primitive_types = set(x + y for x in "uir" for y in ["8", "16", "32", "64"])

    def __init__(self, output_dir = "."):
        self.output_dir = output_dir

    def serialize_string(self, nodes, header = True):
        return "\n".join(header * [self.__render_header()] + [self.__render(node) for node in nodes])

    def serialize(self, nodes, basename):
        path = os.path.join(self.output_dir, basename + ".py")
        out = self.serialize_string(nodes)
        open(path, "w").write(out)

    def __render_header(self):
        return """\
import %s

def bitMaskOr(x, y):
    return x | y

def shiftLeft(x, y):
    return x << y
""" % self.libname

    def __render_include(self, include):
        return "from %s import *\n" % include

    def __render_constant(self, constant):
        return "%s = %s\n" % constant

    def __render_typedef(self, typedef):
        return "%s = %s\n" % (typedef.name, ".".join((self.libname, typedef.type))
                                            if typedef.type in self.primitive_types
                                            else typedef.type)

    def __render_enum_members(self, members):
{'fraction_non_alphanumeric': 0.1303524062144752, 'fraction_numerical': 0.005305039787798408, 'mean_word_length': 2.2116788321167884, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5187881', 'n_tokens_mistral': 825, 'n_tokens_neox': 789, 'n_words': 220}