<commit_before>import os

from pprint import pprint
from django.core.management.base import BaseCommand
from twiggy_goodies.django import LogMixin
from django.conf import settings
from allmychanges.utils import graphite_send
from allmychanges.models import Package, Changelog, Version


def get_stats_from_file():
    """Gets stats dumped to file '.stats'.
    It has simple format like 'name value' on each line

    Usually .stats file used to store such data as number of
    unittests passed/failed, code metrics and parser quality
    metrics.
    """
    filename = os.path.join(settings.PROJECT_ROOT, '.stats')
    if os.path.exists(filename):
        with open(filename) as f:
            lines = f.readlines()
            stats = [line.split(None, 1) for line in lines]
            stats = {name: float(value) for name, value in stats}
            return stats
    return {}


def get_stats():
    stats = get_stats_from_file()

    from rq.scripts.rqinfo import (setup_default_arguments,
                                   parse_args,
                                   setup_redis, Queue)


    args = parse_args()
    setup_default_arguments(args, {})
    setup_redis(args)
    for queue in Queue.all():
        stats['queue.{0}.jobs'.format(queue.name)] = queue.count

    stats['db.packages'] = Package.objects.count()
    stats['db.changelogs'] = Changelog.objects.count()
    stats['db.versions.v1'] = Version.objects.filter(code_version='v1').count()
    stats['db.versions.v2'] = Version.objects.filter(code_version='v2').count()

    return stats


class Command(LogMixin, BaseCommand):
    help = u"""Send stats to graphite Graphite."""

    def handle(self, *args, **options):
        stats = get_stats()
        if args and args[0] == 'dry':
            pprint(stats)
        else:
            graphite_send(**stats)
<commit_msg>Track number of versions, discovered in last minute.<commit_after>import os
import datetime

from pprint import pprint
from django.core.management.base import BaseCommand
from twiggy_goodies.django import LogMixin
from django.conf import settings
from allmychanges.utils import graphite_send
from allmychanges.models import Package, Changelog, Version
from django.utils import timezone


def get_stats_from_file():
    """Gets stats dumped to file '.stats'.
    It has simple format like 'name value' on each line

    Usually .stats file used to store such data as number of
    unittests passed/failed, code metrics and parser quality
    metrics.
    """
    filename = os.path.join(settings.PROJECT_ROOT, '.stats')
    if os.path.exists(filename):
        with open(filename) as f:
            lines = f.readlines()
            stats = [line.split(None, 1) for line in lines]
            stats = {name: float(value) for name, value in stats}
            return stats
    return {}


def get_stats():
    stats = get_stats_from_file()

    from rq.scripts.rqinfo import (setup_default_arguments,
                                   parse_args,
                                   setup_redis, Queue)


    args = parse_args()
    setup_default_arguments(args, {})
    setup_redis(args)
    for queue in Queue.all():
        stats['queue.{0}.jobs'.format(queue.name)] = queue.count

    stats['db.packages'] = Package.objects.count()
    stats['db.changelogs'] = Changelog.objects.count()
    stats['db.versions.v1'] = Version.objects.filter(code_version='v1').count()
    stats['db.versions.v2'] = Version.objects.filter(code_version='v2').count()

    now = timezone.now()
    minute_ago = now - datetime.timedelta(0, 60)

    stats['crawler.discovered.v1.count'] = Version.objects.filter(
        code_version='v1',
        discovered_at__gte=minute_ago).count()
    stats['crawler.discovered.v2.count'] = Version.objects.filter(
        code_version='v2',
        discovered_at__gte=minute_ago).count()

    return stats


class Command(LogMixin, BaseCommand):
    help = u"""Send stats to graphite Graphite."""

    def handle(self, *args, **options):
        stats = get_stats()
        if args and args[0] == 'dry':
            pprint(stats)
        else:
            graphite_send(**stats)

{'fraction_non_alphanumeric': 0.1096649795131357, 'fraction_numerical': 0.005061460592913955, 'mean_word_length': 2.8143382352941178, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '520054', 'n_tokens_mistral': 1241, 'n_tokens_neox': 1204, 'n_words': 323}