<commit_before>from __future__ import division
import logging
import logging.handlers
import datetime
import random


class ConfigRater(object):
    name = 'ConfigRater'
    stopping = False

    def __init__(self, sample):

        self._setup_logging()
        self.logger.debug('Starting ConfigRater for %s' % sample.name if sample is not None else "None")

        self._sample = sample
        self._generatorWorkers = self._sample.config.generatorWorkers

    def __str__(self):
        """Only used for debugging, outputs a pretty printed representation of this output"""
        # Eliminate recursive going back to parent
        temp = dict([ (key, value) for (key, value) in self.__dict__.items() if key != '_c'])
        # return pprint.pformat(temp)
        return ""

    def __repr__(self):
        return self.__str__()

    # loggers can't be pickled due to the lock object, remove them before we try to pickle anything.
    def __getstate__(self):
        temp = self.__dict__
        if getattr(self, 'logger', None):
            temp.pop('logger', None)
        return temp

    def __setstate__(self, d):
        self.__dict__ = d
        self._setup_logging()

    def _setup_logging(self):
        self.logger = logging.getLogger('eventgen')

    def rate(self):
        if type(self._sample.count) != int:
            self._sample.count = int(self._sample.count)
        if type(self._generatorWorkers) != int:
            self._generatorWorkers = int(self._generatorWorkers)
        count = self._sample.count/self._generatorWorkers
        # 5/8/12 CS We've requested not the whole file, so we should adjust count based on
        # hourOfDay, dayOfWeek and randomizeCount configs
        rateFactor = 1.0
        if self._sample.randomizeCount != 0 and self._sample.randomizeCount != None:
            try:
                self.logger.debug("randomizeCount for sample '%s' in app '%s' is %s"
                                  % (self._sample.name, self._sample.app, self._sample.randomizeCount))
                # If we say we're going to be 20% variable, then that means we
                # can be .1% high or .1% low.  Math below does that.
                randBound = round(self._sample.randomizeCount * 1000, 0)
                rand = random.randint(0, randBound)
                randFactor = 1+((-((randBound / 2) - rand)) / 1000)
                self.logger.debug("randFactor for sample '%s' in app '%s' is %s" \
                                % (self._sample.name, self._sample.app, randFactor))
                rateFactor *= randFactor
            except:
                import traceback
                stack = traceback.format_exc()
                self.logger.error("Randomize count failed for sample '%s'.  Stacktrace %s" % (self._sample.name, stack))
        if type(self._sample.hourOfDayRate) == dict:
            try:
                rate = self._sample.hourOfDayRate[str(self._sample.now().hour)]
                self.logger.debug("hourOfDayRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate))
                rateFactor *= rate
            except KeyError:
                import traceback
                stack = traceback.format_exc()
                self.logger.error("Hour of day rate failed for sample '%s'.  Stacktrace %s" % (self._sample.name, stack))
        if type(self._sample.dayOfWeekRate) == dict:
            try:
                weekday = datetime.date.weekday(self._sample.now())
                if weekday == 6:<commit_msg>Fixed mistake in rater<commit_after>from __future__ import division
import logging
import logging.handlers
import datetime
import random


class ConfigRater(object):
    name = 'ConfigRater'
    stopping = False

    def __init__(self, sample):

        self._setup_logging()
        self.logger.debug('Starting ConfigRater for %s' % sample.name if sample is not None else "None")

        self._sample = sample
        self._generatorWorkers = self._sample.config.generatorWorkers

    def __str__(self):
        """Only used for debugging, outputs a pretty printed representation of this output"""
        # Eliminate recursive going back to parent
        temp = dict([ (key, value) for (key, value) in self.__dict__.items() if key != '_c'])
        # return pprint.pformat(temp)
        return ""

    def __repr__(self):
        return self.__str__()

    # loggers can't be pickled due to the lock object, remove them before we try to pickle anything.
    def __getstate__(self):
        temp = self.__dict__
        if getattr(self, 'logger', None):
            temp.pop('logger', None)
        return temp

    def __setstate__(self, d):
        self.__dict__ = d
        self._setup_logging()

    def _setup_logging(self):
        self.logger = logging.getLogger('eventgen')

    def rate(self):
        self._sample.count = int(self._sample.count)
        if self._sample.count == -1:
            self._sample.count = len(self._sample.sampleDict)
        self._generatorWorkers = int(self._generatorWorkers)
        count = self._sample.count/self._generatorWorkers
        # 5/8/12 CS We've requested not the whole file, so we should adjust count based on
        # hourOfDay, dayOfWeek and randomizeCount configs
        rateFactor = 1.0
        if self._sample.randomizeCount:
            try:
                self.logger.debug("randomizeCount for sample '%s' in app '%s' is %s"
                                  % (self._sample.name, self._sample.app, self._sample.randomizeCount))
                # If we say we're going to be 20% variable, then that means we
                # can be .1% high or .1% low.  Math below does that.
                randBound = round(self._sample.randomizeCount * 1000, 0)
                rand = random.randint(0, randBound)
                randFactor = 1+((-((randBound / 2) - rand)) / 1000)
                self.logger.debug("randFactor for sample '%s' in app '%s' is %s" \
                                % (self._sample.name, self._sample.app, randFactor))
                rateFactor *= randFactor
            except:
                import traceback
                stack = traceback.format_exc()
                self.logger.error("Randomize count failed for sample '%s'.  Stacktrace %s" % (self._sample.name, stack))
        if type(self._sample.hourOfDayRate) == dict:
            try:
                rate = self._sample.hourOfDayRate[str(self._sample.now().hour)]
                self.logger.debug("hourOfDayRate for sample '%s' in app '%s' is %s" % (self._sample.name, self._sample.app, rate))
                rateFactor *= rate
            except KeyError:
                import traceback
                stack = traceback.format_exc()
                self.logger.error("Hour of day rate failed for sample '%s'.  Stacktrace %s" % (self._sample.name, stack))
        if type(self._sample.dayOfWeekRate) == dict:
            try:
                weekday = datetime.date.weekday(self._sample.now())
                if weekday == 6:
{'fraction_non_alphanumeric': 0.10332524618238904, 'fraction_numerical': 0.006850292564578279, 'mean_word_length': 2.0589262330859888, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3822365', 'n_tokens_mistral': 1993, 'n_tokens_neox': 1883, 'n_words': 563}