<commit_before>import os
import sys
import configobj
import threading
from warden_logging import log
import logging
from warden_utils import normalize_path

from diamond.server import Server

class DiamondManager:

    def __init__(self, settings):
        self.thread = None
        self.config = None

        configfile = normalize_path(settings.DIAMOND_CONFIG)

        if os.path.exists(configfile):
            self.config = self.build_configObj(configfile)


        else:
            print >> sys.stderr, "ERROR: Config file: %s does not exist." % configfile
            sys.exit(1)

        self.log_diamond = logging.getLogger('diamond')
        self.log_diamond.setLevel(settings.DIAMOND_STDOUT_LEVEL)
        self.log_diamond.propagate = False

#       LOG to STDOUT
        formatter = logging.Formatter('[%(asctime)s][%(levelname)s][%(message)s]')
        streamHandler = logging.StreamHandler(sys.stdout)
        streamHandler.setFormatter(formatter)
        streamHandler.setLevel(settings.DIAMOND_STDOUT_LEVEL)
        self.log_diamond.addHandler(streamHandler)
        self.log_diamond.disabled = False


    def build_configObj(self, config_path):
        config = configobj.ConfigObj(config_path)
        config['configfile'] = config_path

        dr = os.environ['DIAMOND_ROOT']

        self.ensure_var(config['server'],'pidfile', os.path.join(dr,'diamond.pid'))

        self.ensure_var(config['server'],'collectors_path', os.path.join(dr,'collectors'))

        self.ensure_var(config['server'],'collectors_config_path', os.path.join(dr,'collectors','config'))

        self.ensure_var(config['server'],'handlers_config_path', os.path.join(dr,'handlers'))

        return config

    def ensure_var(self, d, v, c):
        if d is None:
            return
        if d[v] is None:
            d[v] = c

    def start(self):
        log.debug("Starting Diamond..")
        self.thread = self.DiamondThread(self.config)
        self.thread.start()

        log.debug("Started Diamond.")

    def stop(self):
        if self.thread.isAlive():
            log.debug("Stopping Diamond..")
            self.thread.stop()
            self.thread.join()
            log.debug("Stopped Diamond.")
        else:
            log.error("Can't stop Diamond if it has not started.")

    def is_active(self):
        if not self.thread:
            return False

        return self.thread.server.running

    class DiamondThread(threading.Thread):

        def __init__(self, config):
            threading.Thread.__init__(self)
            self.server = Server(config)
<commit_msg>Diamond Root ENV, maintains missing diamond conf path entries<commit_after>import os
import sys
import configobj
import threading
from warden_logging import log
import logging
from warden_utils import normalize_path

from diamond.server import Server

class DiamondManager:

    def __init__(self, settings):
        self.thread = None
        self.config = None

        configfile = normalize_path(settings.DIAMOND_CONFIG)

        if os.path.exists(configfile):
            self.config = configobj.ConfigObj(configfile)
            self.config['configfile'] = configfile

            self.ensure_path(self.config['server'],'pid_file', 'diamond.pid')

            self.ensure_path(self.config['server'],'collectors_path', 'collectors','collectors')

            self.ensure_path(self.config['server'],'collectors_config_path', 'collectors','config')

            self.ensure_path(self.config['server'],'handlers_config_path', 'handlers')

        else:
            print >> sys.stderr, "ERROR: Config file: %s does not exist." % configfile
            sys.exit(1)

        self.log_diamond = logging.getLogger('diamond')
        self.log_diamond.setLevel(settings.DIAMOND_STDOUT_LEVEL)
        self.log_diamond.propagate = False

#       LOG to STDOUT
        formatter = logging.Formatter('[%(asctime)s][%(levelname)s][%(message)s]')
        streamHandler = logging.StreamHandler(sys.stdout)
        streamHandler.setFormatter(formatter)
        streamHandler.setLevel(settings.DIAMOND_STDOUT_LEVEL)
        self.log_diamond.addHandler(streamHandler)
        self.log_diamond.disabled = False

    def ensure_path(self, section, var, *args):
        if not var in section:
            try:
                dr = os.environ['DIAMOND_ROOT']
                section[var] = os.path.join(dr, args)
            except KeyError:
                print 'ERROR: Diamond missing path configuration %s[%s] AND $DIAMOND_ROOT has not been set!' % (section, var)
                exit(1)

    def start(self):
        log.debug("Starting Diamond..")
        self.thread = self.DiamondThread(self.config)
        self.thread.start()

        log.debug("Started Diamond.")

    def stop(self):
        if self.thread.isAlive():
            log.debug("Stopping Diamond..")
            self.thread.stop()
            self.thread.join()
            log.debug("Stopped Diamond.")
        else:
            log.error("Can't stop Diamond if it has not started.")

    def is_active(self):
        if not self.thread:
            return False

        return self.thread.server.running

    class DiamondThread(threading.Thread):

        def __init__(self, config):
            threading.Thread.__init__(self)
            self.server = Server(config)

{'fraction_non_alphanumeric': 0.11027616006011648, 'fraction_numerical': 0.0005635919594213789, 'mean_word_length': 2.565974547890154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '86259', 'n_tokens_mistral': 1590, 'n_tokens_neox': 1483, 'n_words': 309}