<commit_before>import pyjs.ast
import sys

if sys.version >= "3":
    basestring = str


def is_string(string):
    return isinstance(string, basestring)


import re

precedence = {
        "||": 5,
        "&&": 6,
        "|": 7,
        "^": 8,
        "&": 9,
        "==": 10,
        "!=": 10,
        "===": 10,
        "!==": 10,
        "<": 11,
        "<=": 11,
        ">": 11,
        ">=": 11,
        "in": 11,
        "instanceof": 11,
        "<<": 12,
        ">>": 12,
        ">>>": 12,
        "+": 13,
        "-": 13,
        "*": 14,
        "/": 14,
        "%": 14,
}


class Serializer(object):
    @classmethod
    def is_bad_identifier(cls, e):
        return isinstance(e, pyjs.ast.Identifier) and \
                not re.match(r'^[_$A-Za-z][_$A-Za-z0-9]*$', e.name)

    @classmethod
    def dump_for_head(cls, e, indent):
        lhs = None
        if isinstance(e.left, pyjs.ast.VariableDeclaration):
            lhs = "%s %s" % (e.left.string,
                             cls.dump_declarators(e.left.declarations,
                                                  indent,
                                                  True))
        else:
            lhs = cls.dump_expr(e.left, indent, 0, True)
        return "for %s(%s in %s)" % ("each " if e.each else "",
                                     lhs,
                                     cls.dump_expr(e.right, indent, 0, False))

    @classmethod
    def wrap_expr(cls, s, cprec, xprec):
        return s if (xprec > cprec) else "(%s)" % s

    @classmethod
    def dump_program(cls, prog):
        lines = [cls.dump_stmt(elem) for elem in prog.body]
        return '\n'.join(lines)

    @classmethod
    def dump_stmt(cls, e, indent=0):
        string = ''
        if isinstance(e, pyjs.ast.BlockStatement):
            string = "%s{\n%s%s}\n" % \
                (" " * indent,
                 "".join([cls.dump_stmt(x, indent + 4) for x in e.body]),
                 " " * indent)
        elif isinstance(e, pyjs.ast.VariableDeclaration):
            string = "%s%s %s;\n" % (" " * indent,
                                  e.kind,
                                  cls.dump_declarators(e.declarations,
                                                       indent,
                                                       False))
        elif isinstance(e, pyjs.ast.ExpressionStatement):<commit_msg>add dummy uneval function<commit_after>import pyjs.ast
import sys

if sys.version >= "3":
    basestring = str


def is_string(string):
    return isinstance(string, basestring)


import re

precedence = {
        "||": 5,
        "&&": 6,
        "|": 7,
        "^": 8,
        "&": 9,
        "==": 10,
        "!=": 10,
        "===": 10,
        "!==": 10,
        "<": 11,
        "<=": 11,
        ">": 11,
        ">=": 11,
        "in": 11,
        "instanceof": 11,
        "<<": 12,
        ">>": 12,
        ">>>": 12,
        "+": 13,
        "-": 13,
        "*": 14,
        "/": 14,
        "%": 14,
}


class Serializer(object):
    @classmethod
    def is_bad_identifier(cls, e):
        return isinstance(e, pyjs.ast.Identifier) and \
                not re.match(r'^[_$A-Za-z][_$A-Za-z0-9]*$', e.name)

    @classmethod
    def dump_for_head(cls, e, indent):
        lhs = None
        if isinstance(e.left, pyjs.ast.VariableDeclaration):
            lhs = "%s %s" % (e.left.string,
                             cls.dump_declarators(e.left.declarations,
                                                  indent,
                                                  True))
        else:
            lhs = cls.dump_expr(e.left, indent, 0, True)
        return "for %s(%s in %s)" % ("each " if e.each else "",
                                     lhs,
                                     cls.dump_expr(e.right, indent, 0, False))

    @classmethod
    def wrap_expr(cls, s, cprec, xprec):
        return s if (xprec > cprec) else "(%s)" % s

    @classmethod
    def uneval(cls, string):
        return "\"%s\"" % string

    @classmethod
    def dump_program(cls, prog):
        lines = [cls.dump_stmt(elem) for elem in prog.body]
        return '\n'.join(lines)

    @classmethod
    def dump_stmt(cls, e, indent=0):
        string = ''
        if isinstance(e, pyjs.ast.BlockStatement):
            string = "%s{\n%s%s}\n" % \
                (" " * indent,
                 "".join([cls.dump_stmt(x, indent + 4) for x in e.body]),
                 " " * indent)
        elif isinstance(e, pyjs.ast.VariableDeclaration):
            string = "%s%s %s;\n" % (" " * indent,
                                  e.kind,
                                  cls.dump_declarators(e.declarations,
                                                       indent,
                                                       False))
        elif isinstance(e, pyjs.ast.ExpressionStatement):
{'fraction_non_alphanumeric': 0.1530737704918033, 'fraction_numerical': 0.019672131147540985, 'mean_word_length': 1.2136054421768707, 'pattern_counts': {'":': 48, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1412391', 'n_tokens_mistral': 1602, 'n_tokens_neox': 1480, 'n_words': 313}