<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Stats Server:

Default Bind IP: localhost
Default Bind Port: 5090

metric data should be be a json string.
Mandator keys:
'metric_name'
'metric_type'

For Metric type counter:
   mandator key: 'count'

"""
import sys
import os
import time
import signal
import socket
import json
from threading import Thread
import multiprocessing
import hashlib
try:
    import pystatsd.pystats_config as pystats_config
except ImportError:
    import pystats_config

try:
    import pystatsd.pystats_log as pystats_log
except ImportError:
    import pystats_log


class TimerMonitor(Thread):
    DEFAULT_INTERVAL = 10

    def __init__(self, duration, metricsmgr):
        super(TimerMonitor, self).__init__()
        self.sleep_interval = duration
        self.metricsmgr = metricsmgr
        self.pylog = pystats_log.Logger(name="TimerMonitor")
        self.logger = self.pylog.logger
        self.logger.info("TimerMonitor Initialized")

    def run(self):
        #pystats_log.print_msg("Timer Monitor start")
        self.logger.debug("Timer Monitor Start")
        while True:
            time.sleep(self.sleep_interval)
            #pystats_log.print_msg("TimerMonitor Wokeup")
            self.logger.debug("TimerMonitor: Wokeup")
            self.metricsmgr.forward_metrics()


class StatsForwarder(object):
    TIMEOUT = 3
    FORWARDERS = {
        'kafka': {'module': 'pystatsd.kafka_publisher',
                  'classname': 'KafkaPublisher'}
    }

    def __init__(self, common_queue):
        self.queue = common_queue
        self.forwarders = {}
        self.pylog = pystats_log.Logger(name="StatsForwarder")
        self.logger = self.pylog.logger

        self.cfg = pystats_config.PyStatConfig()
        if self.cfg.parsedyaml is None:
            self.logger.error("Init: No config parsed")
            return

        self.debug_mode = self.cfg.parsedyaml.get('debug_mode', True)

        for forwarder in self.cfg.parsedyaml['forwarders'].keys():
            fwobj = self.cfg.parsedyaml['forwarders'][forwarder]
            mod = __import__(StatsForwarder.FORWARDERS[forwarder]['module'])
            classobj = getattr(
                mod, StatsForwarder.FORWARDERS[forwarder]['classname'])
            if forwarder == "kafka":
                kafka_broker = fwobj['kafka_broker']
                kafka_apikey = fwobj['kafka_apikey']
                kafka_tenant_id = fwobj['kafka_tenant_id']
                kafka_topic = fwobj['kafka_topic']
                self.forwarders[forwarder] = classobj(kafka_broker,
                                                      kafka_apikey,
                                                      kafka_tenant_id,
                                                      kafka_topic)
        self.logger.info("Initialized")

    def start(self):
        while True:
            objdata = None
            try:
                objdata = self.queue.get(timeout=StatsForwarder.TIMEOUT)
            except multiprocessing.queues.Empty:<commit_msg>FIx the __import__ API<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
Stats Server:

Default Bind IP: localhost
Default Bind Port: 5090

metric data should be be a json string.
Mandator keys:
'metric_name'
'metric_type'

For Metric type counter:
   mandator key: 'count'

"""
import sys
import os
import time
import signal
import socket
import json
from threading import Thread
import multiprocessing
import hashlib
try:
    import pystatsd.pystats_config as pystats_config
except ImportError:
    import pystats_config

try:
    import pystatsd.pystats_log as pystats_log
except ImportError:
    import pystats_log


class TimerMonitor(Thread):
    DEFAULT_INTERVAL = 10

    def __init__(self, duration, metricsmgr):
        super(TimerMonitor, self).__init__()
        self.sleep_interval = duration
        self.metricsmgr = metricsmgr
        self.pylog = pystats_log.Logger(name="TimerMonitor")
        self.logger = self.pylog.logger
        self.logger.info("TimerMonitor Initialized")

    def run(self):
        #pystats_log.print_msg("Timer Monitor start")
        self.logger.debug("Timer Monitor Start")
        while True:
            time.sleep(self.sleep_interval)
            #pystats_log.print_msg("TimerMonitor Wokeup")
            self.logger.debug("TimerMonitor: Wokeup")
            self.metricsmgr.forward_metrics()


class StatsForwarder(object):
    TIMEOUT = 3
    FORWARDERS = {
        'kafka': {'module': 'pystatsd.kafka_publisher',
                  'classname': 'KafkaPublisher'}
    }

    def __init__(self, common_queue):
        self.queue = common_queue
        self.forwarders = {}
        self.pylog = pystats_log.Logger(name="StatsForwarder")
        self.logger = self.pylog.logger

        self.cfg = pystats_config.PyStatConfig()
        if self.cfg.parsedyaml is None:
            self.logger.error("Init: No config parsed")
            return

        self.debug_mode = self.cfg.parsedyaml.get('debug_mode', True)

        for forwarder in self.cfg.parsedyaml['forwarders'].keys():
            fwobj = self.cfg.parsedyaml['forwarders'][forwarder]
            mod = __import__(StatsForwarder.FORWARDERS[forwarder]['module'],
                             fromlist=('kafka_publisher'))

            classobj = getattr(
                mod, StatsForwarder.FORWARDERS[forwarder]['classname'])
            if forwarder == "kafka":
                kafka_broker = fwobj['kafka_broker']
                kafka_apikey = fwobj['kafka_apikey']
                kafka_tenant_id = fwobj['kafka_tenant_id']
                kafka_topic = fwobj['kafka_topic']
                self.forwarders[forwarder] = classobj(kafka_broker,
                                                      kafka_apikey,
                                                      kafka_tenant_id,
                                                      kafka_topic)
        self.logger.info("Initialized")

    def start(self):
        while True:
            objdata = None
            try:
                objdata = self.queue.get(timeout=StatsForwarder.TIMEOUT)
            except multiprocessing.queues.Empty:
{'fraction_non_alphanumeric': 0.09263157894736843, 'fraction_numerical': 0.002591093117408907, 'mean_word_length': 2.188435725348477, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6404197', 'n_tokens_mistral': 1815, 'n_tokens_neox': 1749, 'n_words': 365}