<commit_before>from lib.connector_base import BaseConnector
import datetime

class FixedRangeConnector(BaseConnector):
    def _getQuery(self, document):
        return {
            paramName: document[paramName]
            for paramName in self._groupbyParams
        }

    def push(self, document):
        # 1. build the query
        query = self._getQuery(document)

        # 2. parse datetime
        docDatetime = document[self._timeParamName]

        year = docDatetime.year
        month = docDatetime.month
        day = docDatetime.day
        hour = docDatetime.hour
        minute = docDatetime.minute
        second = docDatetime.second

        dayDate = datetime.datetime(year, month, day)
        hourDate = datetime.datetime(year, month, day, hour)
        minuteDate = datetime.datetime(year, month, day, hour, minute)
        secondDate = datetime.datetime(year, month, day, hour, minute, second)

        # 3. build the $set update
        setUpdate = {
            'days.%s.datetime' % day: dayDate,
            'days.%s.hours.%s.datetime' % (day, hour): hourDate,
            'days.%s.hours.%s.minutes.%s.datetime' % (day, hour, minute): minuteDate,
            'days.%s.hours.%s.minutes.%s.seconds.%s.datetime' % (day, hour, minute, second): secondDate,
        }

        # 4. build the $inc update
        baseIncKeys = [
            '',
            'days.%s.' % day,
            'days.%s.hours.%s.' % (day, hour),
            'days.%s.hours.%s.minutes.%s.' % (day, hour, minute),
            'days.%s.hours.%s.minutes.%s.seconds.%s.' % (day, hour, minute, second),
        ]

        incUpdate = {
            ('%s%s__%s' % (baseIncKey, paramName, aggregateType)): document[paramName] if aggregateType is "sum" else 1
            for baseIncKey in baseIncKeys
            for paramName in self._aggregateParams
            for aggregateType in ['count', 'sum']<commit_msg>Add month year to fixed range connector<commit_after>from lib.connector_base import BaseConnector
import datetime

class FixedRangeConnector(BaseConnector):
    def _getQuery(self, document):
        return {
            paramName: document[paramName]
            for paramName in self._groupbyParams
        }

    def push(self, document):
        # 1. build the query
        query = self._getQuery(document)

        # 2. parse datetime
        docDatetime = document[self._timeParamName]

        year = docDatetime.year
        month = docDatetime.month
        day = docDatetime.day
        hour = docDatetime.hour
        minute = docDatetime.minute
        second = docDatetime.second

        yearDate = datetime.datetime(year, 1, 1)
        monthDate = datetime.datetime(year, month, 1)
        dayDate = datetime.datetime(year, month, day)
        hourDate = datetime.datetime(year, month, day, hour)
        minuteDate = datetime.datetime(year, month, day, hour, minute)
        secondDate = datetime.datetime(year, month, day, hour, minute, second)

        # 3. build the $set update
        setUpdate = {
            'year.%s.datetime' % year: yearDate,
            'year.%s.month.%s.datetime' % (year, month): monthDate,
            'year.%s.month.%s.days.%s.datetime' % (year, month, day): dayDate,
            'year.%s.month.%s.days.%s.hours.%s.datetime' % (year, month, day, hour): hourDate,
            'year.%s.month.%s.days.%s.hours.%s.minutes.%s.datetime' % (year, month, day, hour, minute): minuteDate,
            'year.%s.month.%s.days.%s.hours.%s.minutes.%s.seconds.%s.datetime' % (year, month, day, hour, minute, second): secondDate,
        }

        # 4. build the $inc update
        baseIncKeys = [
            '',
            'year.%s.' % year,
            'year.%s.month.%s.' % (year, month),
            'year.%s.month.%s.days.%s.' % (year, month, day),
            'year.%s.month.%s.days.%s.hours.%s.' % (year, month, day, hour),
            'year.%s.month.%s.days.%s.hours.%s.minutes.%s.' % (year, month, day, hour, minute),
            'year.%s.month.%s.days.%s.hours.%s.minutes.%s.seconds.%s.' % (year, month, day, hour, minute, second),
        ]

        incUpdate = {
            ('%s%s__%s' % (baseIncKey, paramName, aggregateType)): document[paramName] if aggregateType is "sum" else 1
            for baseIncKey in baseIncKeys
            for paramName in self._aggregateParams
            for aggregateType in ['count', 'sum']
{'fraction_non_alphanumeric': 0.13613522156235724, 'fraction_numerical': 0.0029693924166285975, 'mean_word_length': 2.180101670297749, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1180706', 'n_tokens_mistral': 1401, 'n_tokens_neox': 1286, 'n_words': 311}