<commit_before>#
# JavaScript Tools - Parser Module
# License: MPL 1.1/GPL 2.0/LGPL 2.1
# Authors: 
#   - <NAME> <<EMAIL>> (Original JavaScript) (2004)
#   - <NAME> <<EMAIL>> (Python Translation) (2009)
#   - <NAME> <<EMAIL>> (Refactoring Python) (2010)
#

import json


class Node(list):
    def __init__(self, tokenizer=None, type=None, args=[]):
        list.__init__(self)

        if tokenizer:
            token = tokenizer.token
            if token:
                # We may define a custom type but use the same positioning as another token
                # e.g. transform curlys in block nodes, etc.
                self.type = type if type else getattr(token, "type", None)
                self.line = token.line
                
                # Start & end are file positions for error handling.
                self.start = token.start
                self.end = token.end
            
            else:
                self.type = type
                self.line = tokenizer.line
                self.start = None
                self.end = None

            # nodes use a tokenizer for debugging (getSource, filename getter)
            self.tokenizer = tokenizer
            
        elif type:
            self.type = type

        # print "CREATE: %s" % self.type

        for arg in args:
            self.append(arg)


    def getUnrelatedChildren(self):
        """Collects all unrelated children"""
        collection = []
        for child in self:
            if not hasattr(child, "rel"):
                collection.append(child)
            
        return collection
        

    def getChildrenLength(self, filter=True):
        """Number of (per default unrelated) children"""
        count = 0
        for child in self:
            if not filter or not hasattr(child, "rel"):
                count += 1
        return count
            
    
    def remove(self, kid):
        if hasattr(kid, "rel"):
            delattr(self, kid.rel)
            del kid.rel
            
        list.remove(self, kid)
            

    # Always use push to add operands to an expression, to update start and end.
    def append(self, kid, rel=None):
        # kid can be null e.g. [1, , 2].
        if kid:
            if hasattr(kid, "parent"):
                kid.parent.remove(kid)
            
            # Debug
            if not isinstance(kid, Node):
                raise Exception("Invalid kid: %s" % kid)
            
            if hasattr(kid, "tokenizer"):
                if kid.start < self.start:
                    self.start = kid.start

                if self.end < kid.end:
                    self.end = kid.end
                
            kid.parent = self
            
            # alias for function
            if rel != None:
                setattr(self, rel, kid)
                setattr(kid, "rel", rel)

        # Block None kids when they should be related
        if not kid and rel:
            return
            
        return list.append(self, kid)

    
    # Replaces the given kid with the given replacement kid
    def replace(self, kid, repl):
        self[self.index(kid)] = repl
        
        if hasattr(kid, "rel"):
            repl.rel = kid.rel
            setattr(self, kid.rel, repl)
        
        return kid
        

    # Converts the node to XML
    def toXml(self, format=True, indent=0, tab="  "):
        lead = tab * indent if format else ""
        innerLead = tab * (indent+1) if format else ""
        lineBreak = "\n" if format else ""

        relatedChildren = []
        attrsCollection = []
        for name in dir(self):
            # "type" is used as node name - no need to repeat it as an attribute
            # "parent" and "target" are relations to other nodes which are not children - for serialization we ignore them at the moment
            # "rel" is used internally to keep the relation to the parent - used by nodes which need to keep track of specific children
            # "start" and "end" are for debugging only
            if name not in ("type", "parent", "comments", "target", "rel", "start", "end") and name[0] != "_":
                value = getattr(self, name)
                if isinstance(value, Node):
                    if hasattr(value, "rel"):
                        relatedChildren.append(value)

                elif type(value) in (bool, int, long, float, basestring, str, unicode, list):<commit_msg>Fixed some missing details in replace() which resulted in some ugly side effects.<commit_after>#
# JavaScript Tools - Parser Module
# License: MPL 1.1/GPL 2.0/LGPL 2.1
# Authors: 
#   - <NAME> <<EMAIL>> (Original JavaScript) (2004)
#   - <NAME> <<EMAIL>> (Python Translation) (2009)
#   - <NAME> <<EMAIL>> (Refactoring Python) (2010)
#

import json


class Node(list):
    def __init__(self, tokenizer=None, type=None, args=[]):
        list.__init__(self)

        if tokenizer:
            token = tokenizer.token
            if token:
                # We may define a custom type but use the same positioning as another token
                # e.g. transform curlys in block nodes, etc.
                self.type = type if type else getattr(token, "type", None)
                self.line = token.line
                
                # Start & end are file positions for error handling.
                self.start = token.start
                self.end = token.end
            
            else:
                self.type = type
                self.line = tokenizer.line
                self.start = None
                self.end = None

            # nodes use a tokenizer for debugging (getSource, filename getter)
            self.tokenizer = tokenizer
            
        elif type:
            self.type = type

        # print "CREATE: %s" % self.type

        for arg in args:
            self.append(arg)


    def getUnrelatedChildren(self):
        """Collects all unrelated children"""
        collection = []
        for child in self:
            if not hasattr(child, "rel"):
                collection.append(child)
            
        return collection
        

    def getChildrenLength(self, filter=True):
        """Number of (per default unrelated) children"""
        count = 0
        for child in self:
            if not filter or not hasattr(child, "rel"):
                count += 1
        return count
            
    
    def remove(self, kid):
        if hasattr(kid, "rel"):
            delattr(self, kid.rel)
            del kid.rel
            
        list.remove(self, kid)
            

    # Always use push to add operands to an expression, to update start and end.
    def append(self, kid, rel=None):
        # kid can be null e.g. [1, , 2].
        if kid:
            if hasattr(kid, "parent"):
                kid.parent.remove(kid)
            
            # Debug
            if not isinstance(kid, Node):
                raise Exception("Invalid kid: %s" % kid)
            
            if hasattr(kid, "tokenizer"):
                if kid.start < self.start:
                    self.start = kid.start

                if self.end < kid.end:
                    self.end = kid.end
                
            kid.parent = self
            
            # alias for function
            if rel != None:
                setattr(self, rel, kid)
                setattr(kid, "rel", rel)

        # Block None kids when they should be related
        if not kid and rel:
            return
            
        return list.append(self, kid)

    
    # Replaces the given kid with the given replacement kid
    def replace(self, kid, repl):
        self[self.index(kid)] = repl
        
        if hasattr(kid, "rel"):
            repl.rel = kid.rel
            setattr(self, kid.rel, repl)
            delattr(kid, "rel")
            delattr(kid, "parent")
            repl.parent = self
        
        return kid
        

    # Converts the node to XML
    def toXml(self, format=True, indent=0, tab="  "):
        lead = tab * indent if format else ""
        innerLead = tab * (indent+1) if format else ""
        lineBreak = "\n" if format else ""

        relatedChildren = []
        attrsCollection = []
        for name in dir(self):
            # "type" is used as node name - no need to repeat it as an attribute
            # "parent" and "target" are relations to other nodes which are not children - for serialization we ignore them at the moment
            # "rel" is used internally to keep the relation to the parent - used by nodes which need to keep track of specific children
            # "start" and "end" are for debugging only
            if name not in ("type", "parent", "comments", "target", "rel", "start", "end") and name[0] != "_":
                value = getattr(self, name)
                if isinstance(value, Node):
                    if hasattr(value, "rel"):
                        relatedChildren.append(value)

                elif type(value) in (bool, int, long, float, basestring, str, unicode, list):
{'fraction_non_alphanumeric': 0.0937776793622675, 'fraction_numerical': 0.005535872453498672, 'mean_word_length': 1.4680327868852459, 'pattern_counts': {'":': 2, '<': 25, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5646293', 'n_tokens_mistral': 2477, 'n_tokens_neox': 2341, 'n_words': 845}