<commit_before>from dateutil.relativedelta import relativedelta
from django.conf import settings
from django.db.models import Count
import datetime

class InvalidInterval(Exception):
    pass

class DateFieldMissing(Exception):
    pass

class QuerySetMissing(Exception):
    pass

class QuerySetStats(object):
    """
    Generates statistics about a queryset using Django aggregates.  QuerySetStats
    is able to handle snapshots of data (for example this day, week, month, or
    year) or generate time series data suitable for graphing.
    """
    def __init__(self, qs=None, date_field=None, aggregate_field=None, aggregate_class=None):
        self.qs = qs
        self.date_field = date_field
        self.aggregate_field = aggregate_field or getattr(settings, 'QUERYSETSTATS_DEFAULT_AGGREGATE_FIELD', 'id')
        self.aggregate_class = aggregate_class or getattr(settings, 'QUERYSETSTATS_DEFAULT_AGGREGATE_CLASS', Count)
        # MC_TODO: Danger in caching this?
        self.update_today()

    # Aggregates for a specific period of time

    def for_day(self, dt, date_field=None, aggregate_field=None, aggregate_class=None):
        date_field = date_field or self.date_field
        aggregate_class = aggregate_class or self.aggregate_class
        aggregate_field = aggregate_field or self.aggregate_field

        self.check_date_field(date_field)
        self.check_qs()

        kwargs = {
            '%s__year' % date_field : dt.year,
            '%s__month' % date_field : dt.month,
            '%s__day' % date_field : dt.day,
        }
        agg = self.qs.filter(**kwargs).aggregate(agg=aggregate_class(aggregate_field))
        return agg['agg']

    def this_day(self, date_field=None, aggregate_field=None, aggregate_class=None):
        date_field = date_field or self.date_field
        aggregate_class = aggregate_class or self.aggregate_class
        aggregate_field = aggregate_field or self.aggregate_field

        return self.for_day(self.today, date_field, aggregate_field, aggregate_class)

    def for_month(self, dt, date_field=None, aggregate_field=None, aggregate_class=None):
        date_field = date_field or self.date_field
        aggregate_class = aggregate_class or self.aggregate_class
        aggregate_field = aggregate_field or self.aggregate_field

        self.check_date_field(date_field)
        self.check_qs()

        first_day = datetime.date(year=dt.year, month=dt.month, day=1)
        last_day = first_day + relativedelta(day=31)
        return self.get_aggregate(first_day, last_day, date_field, aggregate_field, aggregate_class)

    def this_month(self, date_field=None, aggregate_field=None, aggregate_class=None):
        date_field = date_field or self.date_field
        aggregate_class = aggregate_class or self.aggregate_class

        return self.for_month(self.today, date_field, aggregate_class)

    def for_year(self, dt, date_field=None, aggregate_field=None, aggregate_class=None):
        date_field = date_field or self.date_field
        aggregate_class = aggregate_class or self.aggregate_class
        aggregate_field = aggregate_field or self.aggregate_field

        self.check_date_field(date_field)
        self.check_qs()

        first_day = datetime.date(year=dt.year, month=1, day=1)
        last_day = datetime.date(year=dt.year, month=12, day=31)
        return self.get_aggregate(first_day, last_day, date_field, aggregate_field, aggregate_class)

    def this_year(self, date_field=None, aggregate_field=None, aggregate_class=None):
        date_field = date_field or self.date_field
        aggregate_class = aggregate_class or self.aggregate_class
        aggregate_field = aggregate_field or self.aggregate_field

        return self.for_year(self.today, date_field, aggregate_field, aggregate_class)

    # Aggregate over time intervals

    def time_series(self, start_date, end_date, interval='days', date_field=None, aggregate_field=None, aggregate_class=None):
        if interval not in ('years', 'months', 'weeks', 'days'):
            raise InvalidInterval('Inverval not supported.')

        date_field = date_field or self.date_field
        aggregate_class = aggregate_class or self.aggregate_class
        aggregate_field = aggregate_field or self.aggregate_field
<commit_msg>Removed a bunch of unneccesary boilerplate.<commit_after>from dateutil.relativedelta import relativedelta
from django.conf import settings
from django.db.models import Count
import datetime

class InvalidInterval(Exception):
    pass

class DateFieldMissing(Exception):
    pass

class QuerySetMissing(Exception):
    pass

class QuerySetStats(object):
    """
    Generates statistics about a queryset using Django aggregates.  QuerySetStats
    is able to handle snapshots of data (for example this day, week, month, or
    year) or generate time series data suitable for graphing.
    """
    def __init__(self, qs=None, date_field=None, aggregate_field=None, aggregate_class=None):
        self.qs = qs
        self.date_field = date_field
        self.aggregate_field = aggregate_field or getattr(settings, 'QUERYSETSTATS_DEFAULT_AGGREGATE_FIELD', 'id')
        self.aggregate_class = aggregate_class or getattr(settings, 'QUERYSETSTATS_DEFAULT_AGGREGATE_CLASS', Count)
        # MC_TODO: Danger in caching this?
        self.update_today()

    # Aggregates for a specific period of time

    def for_day(self, dt, date_field=None, aggregate_field=None, aggregate_class=None):
        date_field = date_field or self.date_field
        aggregate_class = aggregate_class or self.aggregate_class
        aggregate_field = aggregate_field or self.aggregate_field

        self.check_date_field(date_field)
        self.check_qs()

        kwargs = {
            '%s__year' % date_field : dt.year,
            '%s__month' % date_field : dt.month,
            '%s__day' % date_field : dt.day,
        }
        agg = self.qs.filter(**kwargs).aggregate(agg=aggregate_class(aggregate_field))
        return agg['agg']

    def this_day(self, date_field=None, aggregate_field=None, aggregate_class=None):
        return self.for_day(self.today, date_field, aggregate_field, aggregate_class)

    def for_month(self, dt, date_field=None, aggregate_field=None, aggregate_class=None):
        date_field = date_field or self.date_field
        aggregate_class = aggregate_class or self.aggregate_class
        aggregate_field = aggregate_field or self.aggregate_field

        self.check_date_field(date_field)
        self.check_qs()

        first_day = datetime.date(year=dt.year, month=dt.month, day=1)
        last_day = first_day + relativedelta(day=31)
        return self.get_aggregate(first_day, last_day, date_field, aggregate_field, aggregate_class)

    def this_month(self, date_field=None, aggregate_field=None, aggregate_class=None):
        return self.for_month(self.today, date_field, aggregate_class)

    def for_year(self, dt, date_field=None, aggregate_field=None, aggregate_class=None):
        date_field = date_field or self.date_field
        aggregate_class = aggregate_class or self.aggregate_class
        aggregate_field = aggregate_field or self.aggregate_field

        self.check_date_field(date_field)
        self.check_qs()

        first_day = datetime.date(year=dt.year, month=1, day=1)
        last_day = datetime.date(year=dt.year, month=12, day=31)
        return self.get_aggregate(first_day, last_day, date_field, aggregate_field, aggregate_class)

    def this_year(self, date_field=None, aggregate_field=None, aggregate_class=None):
        return self.for_year(self.today, date_field, aggregate_field, aggregate_class)

    # Aggregate over time intervals

    def time_series(self, start_date, end_date, interval='days', date_field=None, aggregate_field=None, aggregate_class=None):
        if interval not in ('years', 'months', 'weeks', 'days'):
            raise InvalidInterval('Inverval not supported.')

        date_field = date_field or self.date_field
        aggregate_class = aggregate_class or self.aggregate_class
        aggregate_field = aggregate_field or self.aggregate_field

{'fraction_non_alphanumeric': 0.08011321683485109, 'fraction_numerical': 0.002215111986217081, 'mean_word_length': 3.77497062279671, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2715012', 'n_tokens_mistral': 2471, 'n_tokens_neox': 2395, 'n_words': 553}