<commit_before>from __future__ import division, unicode_literals

import datetime
from collections import Counter, defaultdict

from dateutil import parser
from django.core.exceptions import ImproperlyConfigured
from six.moves import xrange

from kuma.wiki.models import (DocumentDeletionLog, DocumentSpamAttempt,
                              Revision, RevisionAkismetSubmission)
from kuma.wiki.utils import analytics_upageviews


# Rounded to nearby 7-day period for weekly cycles
SPAM_PERIODS = (
    (1, 'period_daily', 'Daily'),
    (7, 'period_weekly', 'Weekly'),
    (28, 'period_monthly', 'Monthly'),
    (91, 'period_quarterly', 'Quarterly'),
)


def date_range(start, end):
    """ Return an iterator providing the dates between `start` and `end`, inclusive.
    """
    if getattr(start, 'date', None) is not None:
        start = start.date()
    if getattr(end, 'date', None) is not None:
        end = end.date()
    days = (end - start).days + 1
    return (start + datetime.timedelta(days=d) for d in xrange(days))


def chunker(seq, size):
    for i in xrange(0, len(seq), size):
        yield seq[i:i + size]


def spam_day_stats(day):
    counts = Counter()
    next_day = day + datetime.timedelta(days=1)

    revs = Revision.objects.filter(
        created__range=(day, next_day)
    ).only('id').prefetch_related('akismet_submissions')
    for rev in revs:
        if any(a.type == 'spam' for a in rev.akismet_submissions.all()):
            counts['published_spam'] += 1<commit_msg>bug 1558255 - xrange is dead (#5490)<commit_after>from __future__ import division, unicode_literals

import datetime
from collections import Counter, defaultdict

from dateutil import parser
from django.core.exceptions import ImproperlyConfigured

from kuma.wiki.models import (DocumentDeletionLog, DocumentSpamAttempt,
                              Revision, RevisionAkismetSubmission)
from kuma.wiki.utils import analytics_upageviews


# Rounded to nearby 7-day period for weekly cycles
SPAM_PERIODS = (
    (1, 'period_daily', 'Daily'),
    (7, 'period_weekly', 'Weekly'),
    (28, 'period_monthly', 'Monthly'),
    (91, 'period_quarterly', 'Quarterly'),
)


def date_range(start, end):
    """ Return an iterator providing the dates between `start` and `end`, inclusive.
    """
    if getattr(start, 'date', None) is not None:
        start = start.date()
    if getattr(end, 'date', None) is not None:
        end = end.date()
    days = (end - start).days + 1
    return (start + datetime.timedelta(days=d) for d in range(days))


def chunker(seq, size):
    for i in range(0, len(seq), size):
        yield seq[i:i + size]


def spam_day_stats(day):
    counts = Counter()
    next_day = day + datetime.timedelta(days=1)

    revs = Revision.objects.filter(
        created__range=(day, next_day)
    ).only('id').prefetch_related('akismet_submissions')
    for rev in revs:
        if any(a.type == 'spam' for a in rev.akismet_submissions.all()):
            counts['published_spam'] += 1
{'fraction_non_alphanumeric': 0.11825449700199867, 'fraction_numerical': 0.010992671552298468, 'mean_word_length': 3.4754098360655736, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5751894', 'n_tokens_mistral': 1006, 'n_tokens_neox': 954, 'n_words': 272}