<commit_before>import simplejson as json
import sys

def compress(node):
    type = node.type
    
    if type in prefixes:
        return prefix(node)
    elif type in postfixes:
        return postfix(node)
    elif type in dividers:
        return divider(node)
    else:
        try:
            return globals()["__" + type](node)
        except KeyError:
            print "Compressor does not support type: %s from line: %s" % (type, node.line)
            print node.toJson()
            sys.exit(1)


dividers = {
    "plus"          : '+',    
    "minus"         : '-',    
    "mul"           : '*',    
    "div"           : '/',    
    "mod"           : '%',
    "dot"           : '.',    
    "property_init" : ":",
}

postfixes = {
    "increment"     : "++",
    "decrement"     : "--"
}

prefixes = {
    "unary_plus"    : "+",
    "unary_minus"   : "-"
}




symbols = {
    "newline"       : '\n',   
    "semicolon"     : ';',    
    "comma"         : ',',    
    "hook"          : '?',    
    "colon"         : ':',    
    "or"            : '||',   
    "and"           : '&&',   
    "bitwise_or"    : '|',    
    "bitwise_xor"   : '^',    
    "bitwise_and"   : '&',    
    "strict_eq"     : '===',  
    "eq"            : '==',   
    "assign"        : '=',    
    "strict_ne"     : '!==',  
    "ne"            : '!=',   
    "lsh"           : '<<',   
    "le"            : '<=',   
    "lt"            : '<',    
    "ursh"          : '>>>',  
    "rsh"           : '>>',   
    "ge"            : '>=',   
    "gt"            : '>',    
    "increment"     : '++',   
    "decrement"     : '--',   
    "not"           : '!',    
    "bitwise_not"   : '~',    
    "left_bracket"  : '[',    
    "right_bracket" : ']',    
    "left_curly"    : '{',    
    "right_curly"   : '}',    
    "left_paren"    : '(',    
    "right_paren"   : ')'
} 


#
# Shared features
#

def divider(node):
    operator = dividers[node.type]
    result = ""
    for child in node:
        result += compress(child) + operator
    result = result[:-len(operator)]
    return result

def postfix(node):
    for child in node:
        return compress(child) + postfixes[node.type]

def prefix(node):
    for child in node:
        return prefixes[node.type] + compress(child)



#
# Main blocks
#

def __script(node):
    result = ""
    for child in node:
        result += compress(child)
        if result[-1] != ";":
            result += ";"
    return result


def __block(node):
    result = "{"
    for child in node:
        result += compress(child) + ";"
    result = result[:-1]
    result += "}"

    return result


def __var(node):
    result = "var "
    for child in node:
        result += compress(child) + ","
    result = result[:-1]

    return result


def __identifier(node):
    result = node.value

    if hasattr(node, "initializer"):
        result += "=%s" % compress(node.initializer)

    return result
    

def __semicolon(node):
    result = ""
    if node.expression:
        result += compress(node.expression)
    return result + ";"


def __call(node):
    result = compress(node[0]) + "("
    for index, child in enumerate(node):
        if index > 0:
            result += compress(child)
    result += ")"
    return result


def __list(node):
    result = ""
    for child in node:
        result += compress(child) + ","
    result = result[:-1]
    return result
        


#
# Primitives
#

def __number(node):
    return "%s" % node.value

def __string(node):
    return json.dumps(node.value)
    
def __true(node):
    return "true"

def __false(node):
    return "false"
    
def __null(node):
    return "null"
    
def __this(node):
    return "this"


#
#
#

def __object_init(node):
    result = "{"
    for child in node:
        result += compress(child)
        result += ","
    result = result[:-1] + "}"
    return result


def __array_init(node):
    result = "["
    for child in node:
        result += compress(child)
        result += ","
    result = result[:-1] + "]"
    return result


def __function(node):
    result = "function"
    
    result += "("
    if len(node.params) > 0:
        for param in node.params:
            result += param + ","
        result = result[:-1]
        
    result += "){"
    
    for child in node.body:
        result += compress(child)
        
    if result.endswith(";"):
        result = result[:-1]
        
    result += "}"
    
    return result
    
    
def __throw(node):
    return "throw " + compress(node.exception)
    
    
def __return(node):
    return "return " + compress(node.value)
    
    
def __new_with_args(node):
    result = ""
    for child in node:
        if result == "":
            result += "new " + compress(child) + "("
        else:
            result += compress(child)
            
    result += ")"
    return result
    
    
def __assign(node):
    result = ""
    for child in node:
        result += compress(child) + "="
    result = result[:-1]
    return result<commit_msg>Added support for if-else<commit_after>import simplejson as json
import sys

def compress(node):
    type = node.type
    
    if type in prefixes:
        return prefix(node)
    elif type in postfixes:
        return postfix(node)
    elif type in dividers:
        return divider(node)
    else:
        try:
            return globals()["__" + type](node)
        except KeyError:
            print "Compressor does not support type: %s from line: %s" % (type, node.line)
            print node.toJson()
            sys.exit(1)


dividers = {
    "plus"          : '+',    
    "minus"         : '-',    
    "mul"           : '*',    
    "div"           : '/',    
    "mod"           : '%',
    "dot"           : '.',    
    "property_init" : ":",
}

postfixes = {
    "increment"     : "++",
    "decrement"     : "--"
}

prefixes = {
    "unary_plus"    : "+",
    "unary_minus"   : "-"
}




symbols = {
    "newline"       : '\n',   
    "semicolon"     : ';',    
    "comma"         : ',',    
    "hook"          : '?',    
    "colon"         : ':',    
    "or"            : '||',   
    "and"           : '&&',   
    "bitwise_or"    : '|',    
    "bitwise_xor"   : '^',    
    "bitwise_and"   : '&',    
    "strict_eq"     : '===',  
    "eq"            : '==',   
    "assign"        : '=',    
    "strict_ne"     : '!==',  
    "ne"            : '!=',   
    "lsh"           : '<<',   
    "le"            : '<=',   
    "lt"            : '<',    
    "ursh"          : '>>>',  
    "rsh"           : '>>',   
    "ge"            : '>=',   
    "gt"            : '>',    
    "increment"     : '++',   
    "decrement"     : '--',   
    "not"           : '!',    
    "bitwise_not"   : '~',    
    "left_bracket"  : '[',    
    "right_bracket" : ']',    
    "left_curly"    : '{',    
    "right_curly"   : '}',    
    "left_paren"    : '(',    
    "right_paren"   : ')'
} 


#
# Shared features
#

def divider(node):
    operator = dividers[node.type]
    result = ""
    for child in node:
        result += compress(child) + operator
    result = result[:-len(operator)]
    return result

def postfix(node):
    for child in node:
        return compress(child) + postfixes[node.type]

def prefix(node):
    for child in node:
        return prefixes[node.type] + compress(child)


#
# Primitives
#

def __number(node):
    return "%s" % node.value

def __string(node):
    return json.dumps(node.value)

def __true(node):
    return "true"

def __false(node):
    return "false"

def __null(node):
    return "null"

def __this(node):
    return "this"
            

#
# Main blocks
#

def __script(node):
    result = ""
    for child in node:
        result += compress(child)
        
        # Verify that each script ends with a semicolon so we can
        # append files after each other without a line feed
        if result[-1] != ";":
            result += ";"
    return result


def __block(node):
    result = "{"
    for child in node:
        result += compress(child) + ";"
    result = result[:-1]
    result += "}"

    return result


def __var(node):
    result = "var "
    for child in node:
        result += compress(child) + ","
    result = result[:-1]

    return result


def __identifier(node):
    result = node.value

    if hasattr(node, "initializer"):
        result += "=%s" % compress(node.initializer)

    return result
    

def __semicolon(node):
    result = ""
    if node.expression:
        result += compress(node.expression)
    return result + ";"


def __call(node):
    result = compress(node[0]) + "("
    for index, child in enumerate(node):
        if index > 0:
            result += compress(child)
    result += ")"
    return result


def __list(node):
    result = ""
    for child in node:
        result += compress(child) + ","
    result = result[:-1]
    return result
        

def __object_init(node):
    result = "{"
    for child in node:
        result += compress(child)
        result += ","
    result = result[:-1] + "}"
    return result


def __array_init(node):
    result = "["
    for child in node:
        result += compress(child)
        result += ","
    result = result[:-1] + "]"
    return result


def __function(node):
    result = "function"
    
    result += "("
    if len(node.params) > 0:
        for param in node.params:
            result += param + ","
        result = result[:-1]
        
    result += "){"
    
    for child in node.body:
        result += compress(child)
        
    if result.endswith(";"):
        result = result[:-1]
        
    result += "}"
    
    return result
    
    
def __throw(node):
    return "throw " + compress(node.exception)
    
    
def __return(node):
    return "return " + compress(node.value)
    
    
def __new_with_args(node):
    result = ""
    for child in node:
        if result == "":
            result += "new " + compress(child) + "("
        else:
            result += compress(child)
            
    result += ")"
    return result
    
    
def __assign(node):
    result = ""
    for child in node:
        result += compress(child) + "="
    result = result[:-1]
    return result
    
    
def __if(node):
    return "if(" + compress(node.condition) + ")" + compress(node.thenPart)
        
    
{'fraction_non_alphanumeric': 0.15664281570295882, 'fraction_numerical': 0.0025140204989363758, 'mean_word_length': 1.5831668331668332, 'pattern_counts': {'":': 6, '<': 11, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4471315', 'n_tokens_mistral': 3446, 'n_tokens_neox': 3391, 'n_words': 736}