<commit_before>#
# JavaScript Tools - Parser Module
# License: MPL 1.1/GPL 2.0/LGPL 2.1
# Authors: 
#   - <NAME> <<EMAIL>> (Original JavaScript) (2004)
#   - <NAME> <<EMAIL>> (Refactoring Python) (2010)
#

import json
import copy

nodeId = 0

class Node(list):
    def __init__(self, tokenizer=None, type=None, args=[]):
        # global nodeId 
        
        list.__init__(self)
        
        self.start = 0
        self.end = 0
        self.filename = None
        
        # Debug
        # self.id = nodeId
        # nodeId += 1

        if tokenizer:
            token = getattr(tokenizer, "token", None)
            if token:
                # We may define a custom type but use the same positioning as another token
                # e.g. transform curlys in block nodes, etc.
                self.type = type if type else getattr(token, "type", None)
                self.line = token.line
                
                # Start & end are file positions for error handling.
                self.start = token.start
                self.end = token.end
            
            else:
                self.type = type
                self.line = tokenizer.line
                self.start = None
                self.end = None

            self.tokenizer = tokenizer
            
        elif type:
            self.type = type

        # print "CREATE: %s" % self.type

        for arg in args:
            self.append(arg)
            
            
    def getUnrelatedChildren(self):
        """Collects all unrelated children"""
        collection = []
        for child in self:
            if not hasattr(child, "rel"):
                collection.append(child)
            
        return collection
        

    def getChildrenLength(self, filter=True):
        """Number of (per default unrelated) children"""
        count = 0
        for child in self:
            if not filter or not hasattr(child, "rel"):
                count += 1
        return count
            
    
    def remove(self, kid):
        if not kid in self:
            raise Exception("Given node is no child!")
        
        if hasattr(kid, "rel"):
            delattr(self, kid.rel)
            del kid.rel
            del kid.parent
            
        list.remove(self, kid)
        
        
    def insert(self, index, kid):
        if index is None:
            return self.append(kid)
            
        if hasattr(kid, "parent"):
            kid.parent.remove(kid)
            
        kid.parent = self

        return list.insert(self, index, kid)
            

    # Always use push to add operands to an expression, to update start and end.
    def append(self, kid, rel=None):
        # kid can be null e.g. [1, , 2].
        if kid:
            if hasattr(kid, "parent"):
                kid.parent.remove(kid)
            
            # Debug
            if not isinstance(kid, Node):
                raise Exception("Invalid kid: %s" % kid)
            
            if hasattr(kid, "tokenizer"):
                if not hasattr(self, "start") or self.start == None or kid.start < self.start:
                    self.start = kid.start

                if not hasattr(self, "end") or self.end == None or self.end < kid.end:
                    self.end = kid.end
                
            kid.parent = self
            
            # alias for function
            if rel != None:
                setattr(self, rel, kid)
                setattr(kid, "rel", rel)

        # Block None kids when they should be related
        if not kid and rel:
            return
            
        return list.append(self, kid)

    
    # Replaces the given kid with the given replacement kid
    def replace(self, kid, repl):
        if repl in self:
            self.remove(repl)
        
        self[self.index(kid)] = repl
        
        if hasattr(kid, "rel"):
            repl.rel = kid.rel
            setattr(self, kid.rel, repl)
            
            # cleanup old kid
            delattr(kid, "rel")
            
            
        elif hasattr(repl, "rel"):
            # delete old relation on new child
            delattr(repl, "rel")

        delattr(kid, "parent")
        repl.parent = self
        
        return kid
        

    # Converts the node to XML
    def toXml(self, format=True, indent=0, tab="  "):
        lead = tab * indent if format else ""
        innerLead = tab * (indent+1) if format else ""
        lineBreak = "\n" if format else ""

        relatedChildren = []
        attrsCollection = []
        for name in dir(self):
            # "type" is used as node name - no need to repeat it as an attribute
            # "parent" and "target" are relations to other nodes which are not children - for serialization we ignore them at the moment
            # "rel" is used internally to keep the relation to the parent - used by nodes which need to keep track of specific children
            # "start" and "end" are for debugging only
            if name not in ("type", "parent", "comments", "target", "rel", "start", "end") and name[0] != "_":
                value = getattr(self, name)
                if isinstance(value, Node):
                    if hasattr(value, "rel"):
                        relatedChildren.append(value)

                elif type(value) in (bool, int, float, str, list, set):
                    if type(value) == bool:
                        value = "true" if value else "false" 
                    elif type(value) in (int, float):
                        value = str(value)
                    elif type(value) == list or type(value) == set:
                        if len(value) == 0:
                            continue
                        try:
                            value = ",".join(value)
                        except TypeError:
                            raise Exception("Invalid attribute list child at: %s" % name)
                                
                    attrsCollection.append('%s=%s' % (name, json.dumps(value)))

        attrs = (" " + " ".join(attrsCollection)) if len(attrsCollection) > 0 else ""
        
        comments = getattr(self, "comments", None)

        if len(self) == 0 and len(relatedChildren) == 0 and (not comments or len(comments) == 0):
            result = "%s<%s%s/>%s" % (lead, self.type, attrs, lineBreak)

        else:
            result = "%s<%s%s>%s" % (lead, self.type, attrs, lineBreak)
            
            if comments:
                for comment in comments:<commit_msg>Added serialization for dictionaries when using toXml()<commit_after>#
# JavaScript Tools - Parser Module
# License: MPL 1.1/GPL 2.0/LGPL 2.1
# Authors: 
#   - <NAME> <<EMAIL>> (Original JavaScript) (2004)
#   - <NAME> <<EMAIL>> (Refactoring Python) (2010)
#

import json
import copy

nodeId = 0

class Node(list):
    def __init__(self, tokenizer=None, type=None, args=[]):
        # global nodeId 
        
        list.__init__(self)
        
        self.start = 0
        self.end = 0
        self.filename = None
        
        # Debug
        # self.id = nodeId
        # nodeId += 1

        if tokenizer:
            token = getattr(tokenizer, "token", None)
            if token:
                # We may define a custom type but use the same positioning as another token
                # e.g. transform curlys in block nodes, etc.
                self.type = type if type else getattr(token, "type", None)
                self.line = token.line
                
                # Start & end are file positions for error handling.
                self.start = token.start
                self.end = token.end
            
            else:
                self.type = type
                self.line = tokenizer.line
                self.start = None
                self.end = None

            self.tokenizer = tokenizer
            
        elif type:
            self.type = type

        # print "CREATE: %s" % self.type

        for arg in args:
            self.append(arg)
            
            
    def getUnrelatedChildren(self):
        """Collects all unrelated children"""
        collection = []
        for child in self:
            if not hasattr(child, "rel"):
                collection.append(child)
            
        return collection
        

    def getChildrenLength(self, filter=True):
        """Number of (per default unrelated) children"""
        count = 0
        for child in self:
            if not filter or not hasattr(child, "rel"):
                count += 1
        return count
            
    
    def remove(self, kid):
        if not kid in self:
            raise Exception("Given node is no child!")
        
        if hasattr(kid, "rel"):
            delattr(self, kid.rel)
            del kid.rel
            del kid.parent
            
        list.remove(self, kid)
        
        
    def insert(self, index, kid):
        if index is None:
            return self.append(kid)
            
        if hasattr(kid, "parent"):
            kid.parent.remove(kid)
            
        kid.parent = self

        return list.insert(self, index, kid)
            

    # Always use push to add operands to an expression, to update start and end.
    def append(self, kid, rel=None):
        # kid can be null e.g. [1, , 2].
        if kid:
            if hasattr(kid, "parent"):
                kid.parent.remove(kid)
            
            # Debug
            if not isinstance(kid, Node):
                raise Exception("Invalid kid: %s" % kid)
            
            if hasattr(kid, "tokenizer"):
                if not hasattr(self, "start") or self.start == None or kid.start < self.start:
                    self.start = kid.start

                if not hasattr(self, "end") or self.end == None or self.end < kid.end:
                    self.end = kid.end
                
            kid.parent = self
            
            # alias for function
            if rel != None:
                setattr(self, rel, kid)
                setattr(kid, "rel", rel)

        # Block None kids when they should be related
        if not kid and rel:
            return
            
        return list.append(self, kid)

    
    # Replaces the given kid with the given replacement kid
    def replace(self, kid, repl):
        if repl in self:
            self.remove(repl)
        
        self[self.index(kid)] = repl
        
        if hasattr(kid, "rel"):
            repl.rel = kid.rel
            setattr(self, kid.rel, repl)
            
            # cleanup old kid
            delattr(kid, "rel")
            
            
        elif hasattr(repl, "rel"):
            # delete old relation on new child
            delattr(repl, "rel")

        delattr(kid, "parent")
        repl.parent = self
        
        return kid
        

    # Converts the node to XML
    def toXml(self, format=True, indent=0, tab="  "):
        lead = tab * indent if format else ""
        innerLead = tab * (indent+1) if format else ""
        lineBreak = "\n" if format else ""

        relatedChildren = []
        attrsCollection = []
        for name in dir(self):
            # "type" is used as node name - no need to repeat it as an attribute
            # "parent" and "target" are relations to other nodes which are not children - for serialization we ignore them at the moment
            # "rel" is used internally to keep the relation to the parent - used by nodes which need to keep track of specific children
            # "start" and "end" are for debugging only
            if name not in ("type", "parent", "comments", "target", "rel", "start", "end") and name[0] != "_":
                value = getattr(self, name)
                if isinstance(value, Node):
                    if hasattr(value, "rel"):
                        relatedChildren.append(value)

                elif type(value) in (bool, int, float, str, list, set, dict):
                    if type(value) == bool:
                        value = "true" if value else "false" 
                    elif type(value) in (int, float):
                        value = str(value)
                    elif type(value) in (list, set, dict):
                        if type(value) == dict:
                            value = value.keys()
                        if len(value) == 0:
                            continue
                        try:
                            value = ",".join(value)
                        except TypeError:
                            raise Exception("Invalid attribute list child at: %s" % name)
                            
                    attrsCollection.append('%s=%s' % (name, json.dumps(value)))

        attrs = (" " + " ".join(attrsCollection)) if len(attrsCollection) > 0 else ""
        
        comments = getattr(self, "comments", None)

        if len(self) == 0 and len(relatedChildren) == 0 and (not comments or len(comments) == 0):
            result = "%s<%s%s/>%s" % (lead, self.type, attrs, lineBreak)

        else:
            result = "%s<%s%s>%s" % (lead, self.type, attrs, lineBreak)
            
            if comments:
                for comment in comments:
{'fraction_non_alphanumeric': 0.0967448902346707, 'fraction_numerical': 0.004542013626040878, 'mean_word_length': 1.3144709180098109, 'pattern_counts': {'":': 2, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5247677', 'n_tokens_mistral': 3590, 'n_tokens_neox': 3413, 'n_words': 1155}