<commit_before>'''Graphite Uploader plugin that sends aggregated data to Graphite server'''

from Tank.Plugins.Aggregator import AggregateResultListener, AggregatorPlugin
from tankcore import AbstractPlugin
import logging
import socket
import string
import time
import datetime
import os

class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener):
    '''Graphite data uploader'''
    
    SECTION = 'graphite'

    @staticmethod
    def get_key():
        return __file__
    
    def __init__(self, core):
        AbstractPlugin.__init__(self, core)
        self.graphite_client = None

    def get_available_options(self):
        return ["address", "port", "prefix", "web_port"]

    def start_test(self):
        start_time = datetime.datetime.now()
        self.start_time = start_time.strftime("%H:%M%%20%Y%m%d")

    def end_test(self, retcode):
        end_time = datetime.datetime.now() + datetime.timedelta(minutes = 1)
        self.end_time = end_time.strftime("%H:%M%%20%Y%m%d")
        return retcode

    def configure(self):
        '''Read configuration'''
        self.address = self.get_option("address", "")
        if self.address == "": 
            self.log.warning("Graphite uploader is not configured and will not send any data")
        else:
            port = self.get_option("port", "2003")
            self.web_port = self.get_option("web_port", "8080")
            self.prefix = self.get_option("prefix", "one_sec.yandex_tank")
            self.template = self.get_option("template", os.path.dirname(__file__) + "/graphite.tpl")
            self.graphite_client = GraphiteClient(self.prefix, self.address, port)
            aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
            aggregator.add_result_listener(self)
            
    def aggregate_second(self, data):
        """
        @data: SecondAggregateData
        """
        if self.graphite_client:
            results = {}
            overall = GraphiteUploaderPlugin.__flatten(data.overall.__dict__, "overall")
            cumulative = GraphiteUploaderPlugin.__flatten(data.cumulative.__dict__, "cumulative")
            results.update(overall)
            results.update(cumulative)
            for marker in data.cases.keys():
                results.update(GraphiteUploaderPlugin.__flatten(data.cases[marker].__dict__, 'markers.%s' % marker))
            self.graphite_client.submit(results)

    def post_process(self, retcode):
        if self.graphite_client:
            template = open(self.template, 'r').read()
            graphite_html = self.core.mkstemp(".html", "graphite_")
            self.core.add_artifact_file(graphite_html)
            with open(graphite_html, 'w') as graphite_html_file:
                graphite_html_file.write(
                    template.format(
                        host='%s:%s' % (self.address, self.web_port),
                        width=1000,
                        height=400,
                        start_time=self.start_time,
                        end_time=self.end_time,
                        prefix=self.prefix,
                        web_port=self.web_port,
                    )
                )

    @staticmethod
    def __flatten(dic, prefix):
        '''recursively pass through a dict and flatten it\'s "internal" dicts'''
        results = {}
        if dic != None:
            try:
                for key in dic.keys():
                    if type(dic[key]) in [float, int]:
                        results["%s.%s" % (prefix, str(key).translate(string.maketrans(".", "_")))] = dic[key]
                    elif type(dic[key] in [dict]):
                        results.update(GraphiteUploaderPlugin.__flatten(dic[key], "%s.%s" % (prefix, key.translate(string.maketrans(".", "_")))))
            except AttributeError:
                pass
        return results


    
class GraphiteClient(object):<commit_msg>fix bug: web_port was added twice<commit_after>'''Graphite Uploader plugin that sends aggregated data to Graphite server'''

from Tank.Plugins.Aggregator import AggregateResultListener, AggregatorPlugin
from tankcore import AbstractPlugin
import logging
import socket
import string
import time
import datetime
import os

class GraphiteUploaderPlugin(AbstractPlugin, AggregateResultListener):
    '''Graphite data uploader'''
    
    SECTION = 'graphite'

    @staticmethod
    def get_key():
        return __file__
    
    def __init__(self, core):
        AbstractPlugin.__init__(self, core)
        self.graphite_client = None

    def get_available_options(self):
        return ["address", "port", "prefix", "web_port"]

    def start_test(self):
        start_time = datetime.datetime.now()
        self.start_time = start_time.strftime("%H:%M%%20%Y%m%d")

    def end_test(self, retcode):
        end_time = datetime.datetime.now() + datetime.timedelta(minutes = 1)
        self.end_time = end_time.strftime("%H:%M%%20%Y%m%d")
        return retcode

    def configure(self):
        '''Read configuration'''
        self.address = self.get_option("address", "")
        if self.address == "": 
            self.log.warning("Graphite uploader is not configured and will not send any data")
        else:
            port = self.get_option("port", "2003")
            self.web_port = self.get_option("web_port", "8080")
            self.prefix = self.get_option("prefix", "one_sec.yandex_tank")
            self.template = self.get_option("template", os.path.dirname(__file__) + "/graphite.tpl")
            self.graphite_client = GraphiteClient(self.prefix, self.address, port)
            aggregator = self.core.get_plugin_of_type(AggregatorPlugin)
            aggregator.add_result_listener(self)
            
    def aggregate_second(self, data):
        """
        @data: SecondAggregateData
        """
        if self.graphite_client:
            results = {}
            overall = GraphiteUploaderPlugin.__flatten(data.overall.__dict__, "overall")
            cumulative = GraphiteUploaderPlugin.__flatten(data.cumulative.__dict__, "cumulative")
            results.update(overall)
            results.update(cumulative)
            for marker in data.cases.keys():
                results.update(GraphiteUploaderPlugin.__flatten(data.cases[marker].__dict__, 'markers.%s' % marker))
            self.graphite_client.submit(results)

    def post_process(self, retcode):
        if self.graphite_client:
            template = open(self.template, 'r').read()
            graphite_html = self.core.mkstemp(".html", "graphite_")
            self.core.add_artifact_file(graphite_html)
            with open(graphite_html, 'w') as graphite_html_file:
                graphite_html_file.write(
                    template.format(
                        host='%s' % self.address,
                        width=1000,
                        height=400,
                        start_time=self.start_time,
                        end_time=self.end_time,
                        prefix=self.prefix,
                        web_port=self.web_port,
                    )
                )

    @staticmethod
    def __flatten(dic, prefix):
        '''recursively pass through a dict and flatten it\'s "internal" dicts'''
        results = {}
        if dic != None:
            try:
                for key in dic.keys():
                    if type(dic[key]) in [float, int]:
                        results["%s.%s" % (prefix, str(key).translate(string.maketrans(".", "_")))] = dic[key]
                    elif type(dic[key] in [dict]):
                        results.update(GraphiteUploaderPlugin.__flatten(dic[key], "%s.%s" % (prefix, key.translate(string.maketrans(".", "_")))))
            except AttributeError:
                pass
        return results


    
class GraphiteClient(object):
{'fraction_non_alphanumeric': 0.11274635270028155, 'fraction_numerical': 0.005119017148707448, 'mean_word_length': 2.1716720779220777, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5373277', 'n_tokens_mistral': 2256, 'n_tokens_neox': 2147, 'n_words': 431}