<commit_before>from node import Node, NodeList
from .qg_profiler import QGProfiler
from .helper import get_real_file_path, get_file_type, xml_scanner, read_attributes_from_xml, merge_attributes
import glob
import json

class QGProfileAggregator(object):
    def __init__(self, in_file_path, out_file_path):
        self.root_node = Node('i_am_root', None, {})
        self.in_file_path = get_real_file_path(in_file_path)
        get_file_type(out_file_path)
        self.out_file_path = get_real_file_path(out_file_path)

    def add_json(self, _json):
        new_node = self.make_node_from_json(_json, self.root_node)
        new_node_list = NodeList()
        new_node_list.append(new_node)
        self.merge_node_list_to_node(self.root_node, new_node_list)

    def merge_node_list_to_node(self, main_node, node_list):
        for node in node_list:
            index = main_node.is_child_in_children(node.get_name())
            if index == -1:
                main_node.add_child(node)
            else:
                existing_node = main_node.get_child(index)
                existing_node.increment_value_by(node.get_value())
                existing_node.increment_count_by(node.get_count())
                existing_node.set_aggregate_attr(merge_attributes(node.get_aggregate_attr(), existing_node.get_aggregate_attr()))
                existing_node.update_over_head(node.get_over_head())
                self.merge_node_list_to_node(existing_node, node.get_children())

    def make_node_from_json(self, _json, parent_node):
        name = _json['name']
        value = _json['value']
        count = _json['count']
        children = _json['children']
        attributes = _json['attributes']
        over_head = _json['overhead']
        new_node = Node(name, parent_node, attributes)
        new_node.set_value(value)
        new_node.set_count(count)
        new_node.set_over_head(over_head)

        for child in children:
            child_node = self.make_node_from_json(child, new_node)
            new_node.add_child(child_node)
        return new_node

    def add_xml(self, _xml):
        current_node = self.root_node
        xml_gen = xml_scanner(_xml)
        for each in xml_gen:
            if each[0] == 'START':
                name = str(each[2]['name'])
                value = float(each[2]['value'])
                count = int(each[2]['count'])
                over_head = float(each[2]['overhead'])
                attributes = read_attributes_from_xml(each[2]['attributes'])
                index = current_node.is_child_in_children(name)
                if index == -1:
                    new_node = Node(name, current_node, attributes)
                    new_node.set_value(value)
                    new_node.set_count(count)
                    new_node.set_over_head(over_head)
                    current_node.add_child(new_node)
                    current_node = new_node<commit_msg>default attributes and overhead being passed to support older version<commit_after>from node import Node, NodeList
from .qg_profiler import QGProfiler
from .helper import get_real_file_path, get_file_type, xml_scanner, read_attributes_from_xml, merge_attributes
import glob
import json

class QGProfileAggregator(object):
    def __init__(self, in_file_path, out_file_path):
        self.root_node = Node('i_am_root', None, {})
        self.in_file_path = get_real_file_path(in_file_path)
        get_file_type(out_file_path)
        self.out_file_path = get_real_file_path(out_file_path)

    def add_json(self, _json):
        new_node = self.make_node_from_json(_json, self.root_node)
        new_node_list = NodeList()
        new_node_list.append(new_node)
        self.merge_node_list_to_node(self.root_node, new_node_list)

    def merge_node_list_to_node(self, main_node, node_list):
        for node in node_list:
            index = main_node.is_child_in_children(node.get_name())
            if index == -1:
                main_node.add_child(node)
            else:
                existing_node = main_node.get_child(index)
                existing_node.increment_value_by(node.get_value())
                existing_node.increment_count_by(node.get_count())
                existing_node.set_aggregate_attr(merge_attributes(node.get_aggregate_attr(), existing_node.get_aggregate_attr()))
                existing_node.update_over_head(node.get_over_head())
                self.merge_node_list_to_node(existing_node, node.get_children())

    def make_node_from_json(self, _json, parent_node):
        name = _json['name']
        value = _json['value']
        count = _json['count']
        children = _json['children']
        attributes = _json.get('attributes', {})
        over_head = _json.get('overhead', 0)
        new_node = Node(name, parent_node, attributes)
        new_node.set_value(value)
        new_node.set_count(count)
        new_node.set_over_head(over_head)

        for child in children:
            child_node = self.make_node_from_json(child, new_node)
            new_node.add_child(child_node)
        return new_node

    def add_xml(self, _xml):
        current_node = self.root_node
        xml_gen = xml_scanner(_xml)
        for each in xml_gen:
            if each[0] == 'START':
                name = str(each[2]['name'])
                value = float(each[2]['value'])
                count = int(each[2]['count'])
                over_head = float(each[2].get('overhead', 0))
                attributes = read_attributes_from_xml(each[2].get('attributes', {}))
                index = current_node.is_child_in_children(name)
                if index == -1:
                    new_node = Node(name, current_node, attributes)
                    new_node.set_value(value)
                    new_node.set_count(count)
                    new_node.set_over_head(over_head)
                    current_node.add_child(new_node)
                    current_node = new_node
{'fraction_non_alphanumeric': 0.09170305676855896, 'fraction_numerical': 0.0030231776956667787, 'mean_word_length': 2.49882491186839, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 18}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6120347', 'n_tokens_mistral': 1869, 'n_tokens_neox': 1819, 'n_words': 294}