<commit_before>'''
Copyright (C) 2005-2012 Splunk Inc. All Rights Reserved.
'''
from __future__ import division

import sys, os
path_prepend = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'lib')
sys.path.append(path_prepend)

import logging
import threading
import time
import datetime
from select import select
from eventgenconfig import Config
from timeparser import timeDelta2secs

class Timer(threading.Thread):
    # Added by CS 5/7/12 to emulate threading.Timer
    def __init__(self, time, sample):
        self.time = time
        self.stopping = False
        
        self.sample = sample
        threading.Thread.__init__(self)

    def run(self):
        if self.sample.delay > 0:
            logger.info("Sample set to delay %s, sleeping." % s.delay)
            time.sleep(self.sample.delay)
        while (1):
            if not self.stopping:
                startTime = datetime.datetime.now()
                self.sample.gen()
                endTime = datetime.datetime.now()
                timeDiff = endTime - startTime

                timeDiffFrac = "%s.%s" % (timeDiff.seconds, timeDiff.microseconds)
                logger.info("Generation of sample '%s' in app '%s' completed in %s seconds" \
                            % (self.sample.name, self.sample.app, timeDiffFrac) )

                timeDiff = timeDelta2secs(timeDiff)
                wholeIntervals = timeDiff / self.sample.interval
                partialInterval = timeDiff % self.sample.interval

                if wholeIntervals > 1:
                    logger.warn("Generation of sample '%s' in app '%s' took longer than interval (%s seconds vs. %s seconds); consider adjusting interval" \
                                % (self.sample.name, self.sample.app, timeDiff, self.sample.interval) )

                partialInterval = self.sample.interval - partialInterval
                logger.debug("Generation of sample '%s' in app '%s' sleeping for %s seconds" \
                            % (self.sample.name, self.sample.app, partialInterval) )

                ## Sleep for partial interval
                time.sleep(partialInterval)
            else:
                sys.exit(0)

    def stop(self):
        self.stopping = True
                     
            
# Copied from http://danielkaes.wordpress.com/2009/06/04/how-to-catch-kill-events-with-python/
def set_exit_handler(func):
	if os.name == "nt":
		try:
			import win32api
			win32api.SetConsoleCtrlHandler(func, True)
		except ImportError:
			version = ".".join(map(str, sys.version_info[:2]))
			raise Exception("pywin32 not installed for Python " + version)
	else:
		import signal
		signal.signal(signal.SIGTERM, func)
		signal.signal(signal.SIGINT, func)
    
def handle_exit(sig=None, func=None):
    print '\n\nCaught kill, exiting...'
    for sampleTimer in sampleTimers:
        sampleTimer.stop()
    sys.exit(0)
    		

if __name__ == '__main__':
    debug = False
    c = Config()
    # Logger is setup by Config, just have to get an instance
    logger = logging.getLogger('eventgen')
    logger.info('Starting eventgen')
    
    # 5/6/12 CS use select to listen for input on stdin
    # if we timeout, assume we're not splunk embedded
    # Only support standalone mode on Unix due to limitation with select()
    if os.name != "nt":
        rlist, _, _ = select([sys.stdin], [], [], 5)
        if rlist:
            sessionKey = sys.stdin.readline().strip()
        else:
            sessionKey = ''
    else:
        sessionKey = sys.stdin.readline().strip()
    
    if sessionKey == 'debug':
        c.makeSplunkEmbedded(debug=True)
    elif len(sessionKey) > 0:
        c.makeSplunkEmbedded(sessionKey=sessionKey)
        
    c.parse()

    sampleTimers = []
        
    if c.debug:
        logger.info('Entering debug (single iteration) mode')
<commit_msg>Trying to be able to exit properly.  Threads suck.<commit_after>'''
Copyright (C) 2005-2012 Splunk Inc. All Rights Reserved.
'''
from __future__ import division

import sys, os
path_prepend = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'lib')
sys.path.append(path_prepend)

import logging
import threading
import time
import datetime
from select import select
from eventgenconfig import Config
from timeparser import timeDelta2secs

class Timer(threading.Thread):
    time = None
    stopping = None
    interruptcatcher = None
    
    # Added by CS 5/7/12 to emulate threading.Timer
    def __init__(self, time, sample=None, interruptcatcher=None):
        self.time = time
        self.stopping = False
        self.interruptcatcher = interruptcatcher
        
        self.sample = sample
        threading.Thread.__init__(self)

    def run(self):
        if self.sample.delay > 0:
            logger.info("Sample set to delay %s, sleeping." % s.delay)
            time.sleep(self.sample.delay)
        while (1):
            if not self.stopping:
                if not self.interruptcatcher:
                    startTime = datetime.datetime.now()
                    self.sample.gen()
                    endTime = datetime.datetime.now()
                    timeDiff = endTime - startTime

                    timeDiffFrac = "%s.%s" % (timeDiff.seconds, timeDiff.microseconds)
                    logger.info("Generation of sample '%s' in app '%s' completed in %s seconds" \
                                % (self.sample.name, self.sample.app, timeDiffFrac) )

                    timeDiff = timeDelta2secs(timeDiff)
                    wholeIntervals = timeDiff / self.sample.interval
                    partialInterval = timeDiff % self.sample.interval

                    if wholeIntervals > 1:
                        logger.warn("Generation of sample '%s' in app '%s' took longer than interval (%s seconds vs. %s seconds); consider adjusting interval" \
                                    % (self.sample.name, self.sample.app, timeDiff, self.sample.interval) )

                    partialInterval = self.sample.interval - partialInterval
                    logger.debug("Generation of sample '%s' in app '%s' sleeping for %s seconds" \
                                % (self.sample.name, self.sample.app, partialInterval) )

                    ## Sleep for partial interval
                    time.sleep(partialInterval)
                else:
                    time.sleep(self.time)
            else:
                sys.exit(0)

    def stop(self):
        self.stopping = True
                     
            
# Copied from http://danielkaes.wordpress.com/2009/06/04/how-to-catch-kill-events-with-python/
def set_exit_handler(func):
	if os.name == "nt":
		try:
			import win32api
			win32api.SetConsoleCtrlHandler(func, True)
		except ImportError:
			version = ".".join(map(str, sys.version_info[:2]))
			raise Exception("pywin32 not installed for Python " + version)
	else:
		import signal
		signal.signal(signal.SIGTERM, func)
		signal.signal(signal.SIGINT, func)
    
def handle_exit(sig=None, func=None):
    print '\n\nCaught kill, exiting...'
    for sampleTimer in sampleTimers:
        sampleTimer.stop()
    sys.exit(0)
    		

if __name__ == '__main__':
    debug = False
    c = Config()
    # Logger is setup by Config, just have to get an instance
    logger = logging.getLogger('eventgen')
    logger.info('Starting eventgen')
    
    # 5/6/12 CS use select to listen for input on stdin
    # if we timeout, assume we're not splunk embedded
    # Only support standalone mode on Unix due to limitation with select()
    if os.name != "nt":
        rlist, _, _ = select([sys.stdin], [], [], 5)
        if rlist:
            sessionKey = sys.stdin.readline().strip()
        else:
            sessionKey = ''
    else:
        sessionKey = sys.stdin.readline().strip()
    
    if sessionKey == 'debug':
        c.makeSplunkEmbedded(debug=True)
    elif len(sessionKey) > 0:
        c.makeSplunkEmbedded(sessionKey=sessionKey)
        
    c.parse()

    sampleTimers = []
        
    if c.debug:
        logger.info('Entering debug (single iteration) mode')

    # Hopefully this will catch interrupts, signals, etc
    # To allow us to stop gracefully
    t = Timer(1.0, interruptcatcher=True)

{'fraction_non_alphanumeric': 0.10308402136959689, 'fraction_numerical': 0.00995628946090335, 'mean_word_length': 2.094290007513148, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '569032', 'n_tokens_mistral': 2464, 'n_tokens_neox': 2296, 'n_words': 626}