<commit_before>#
# JavaScript Tools - Optimizer for local variable names
# Copyright 2010 <NAME>
#

from js.tokenizer.Tokenizer import keywords
from copy import copy
import string, logging

__all__ = ["optimize"]

empty = ("null", "this", "true", "false", "number", "string", "regexp")


#
# Public API
#

def optimize(node):
    undefines = __scanScope(node)
    __patch(node)


#
# Implementation
#

def __baseEncode(num, alphabet=string.ascii_letters):
    if (num == 0):
        return alphabet[0]
    arr = []
    base = len(alphabet)
    while num:
        rem = num % base
        num = num // base
        arr.append(alphabet[rem])
    arr.reverse()
    return "".join(arr)



def __scanNode(node, delcares, uses):
    if node.type == "function":
        functionName = getattr(node, "name", None)
        if functionName:
            delcares.add(functionName)
    
    elif node.type == "declaration":
        varName = getattr(node, "name", None)
        if varName != None:
            delcares.add(varName)
            
    elif node.type == "identifier":
        if node.parent.type == "list" and getattr(node.parent, "rel", None) == "params":
            pass
            
        elif node.parent.type != "dot" or node.parent.index(node) == 0:
            if node.value in uses:
                uses[node.value] += 1
            else:
                uses[node.value] = 1
    
    if node.type == "script":
        childUndefines = __scanScope(node)
        for name in childUndefines:
            if name in uses:
                uses[name] += childUndefines[name]
            else:
                uses[name] = childUndefines[name]
                
    # Treat catch blocks like new scopes
    elif node.type == "block" and getattr(node, "rel") == "block" and node.parent.type == "catch":
        childUndefines = __scanScope(node)
        for name in childUndefines:
            if name in uses:
                uses[name] += childUndefines[name]
            else:
                uses[name] = childUndefines[name]        
        
    else:
        for child in node:
            __scanNode(child, delcares, uses)



def __scanScope(node):
    defines = set()
    uses = {}
    
    # Add params to declaration list
    __addParams(node, defines)
    __addExceptions(node, defines)

    # Process children
    for child in node:
        __scanNode(child, defines, uses)
    
    # Look for used varibles which have not been defined
    # Might be a part of a closure or just a mistake
    undefines = {}
    for name in uses:
        if name not in defines:
            undefines[name] = uses[name]

    print("Quit Scope [Line:%s]" % node.line)
    print("- Defines:", defines)
    print("- Uses:", uses)
    print("- Undefines:", undefines)<commit_msg>Disabled debug output<commit_after>#
# JavaScript Tools - Optimizer for local variable names
# Copyright 2010 <NAME>
#

from js.tokenizer.Tokenizer import keywords
from copy import copy
import string, logging

__all__ = ["optimize"]

empty = ("null", "this", "true", "false", "number", "string", "regexp")


#
# Public API
#

def optimize(node):
    undefines = __scanScope(node)
    __patch(node)


#
# Implementation
#

def __baseEncode(num, alphabet=string.ascii_letters):
    if (num == 0):
        return alphabet[0]
    arr = []
    base = len(alphabet)
    while num:
        rem = num % base
        num = num // base
        arr.append(alphabet[rem])
    arr.reverse()
    return "".join(arr)



def __scanNode(node, delcares, uses):
    if node.type == "function":
        functionName = getattr(node, "name", None)
        if functionName:
            delcares.add(functionName)
    
    elif node.type == "declaration":
        varName = getattr(node, "name", None)
        if varName != None:
            delcares.add(varName)
            
    elif node.type == "identifier":
        if node.parent.type == "list" and getattr(node.parent, "rel", None) == "params":
            pass
            
        elif node.parent.type != "dot" or node.parent.index(node) == 0:
            if node.value in uses:
                uses[node.value] += 1
            else:
                uses[node.value] = 1
    
    if node.type == "script":
        childUndefines = __scanScope(node)
        for name in childUndefines:
            if name in uses:
                uses[name] += childUndefines[name]
            else:
                uses[name] = childUndefines[name]
                
    # Treat catch blocks like new scopes
    elif node.type == "block" and getattr(node, "rel") == "block" and node.parent.type == "catch":
        childUndefines = __scanScope(node)
        for name in childUndefines:
            if name in uses:
                uses[name] += childUndefines[name]
            else:
                uses[name] = childUndefines[name]        
        
    else:
        for child in node:
            __scanNode(child, delcares, uses)



def __scanScope(node):
    defines = set()
    uses = {}
    
    # Add params to declaration list
    __addParams(node, defines)
    __addExceptions(node, defines)

    # Process children
    for child in node:
        __scanNode(child, defines, uses)
    
    # Look for used varibles which have not been defined
    # Might be a part of a closure or just a mistake
    undefines = {}
    for name in uses:
        if name not in defines:
            undefines[name] = uses[name]

    # print("Quit Scope [Line:%s]" % node.line)
    # print("- Defines:", defines)
    # print("- Uses:", uses)
    # print("- Undefines:", undefines)
{'fraction_non_alphanumeric': 0.11430625449317038, 'fraction_numerical': 0.003235082674335011, 'mean_word_length': 2.056013179571664, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4313288', 'n_tokens_mistral': 1713, 'n_tokens_neox': 1653, 'n_words': 481}