<commit_before>from __future__ import division
from django.core.management.base import BaseCommand
from django.utils import timezone
from thezombies.models import (Agency, Probe)

REPORT_DATE_FORMATTER = u"{:%Y-%m-%d %I:%M%p %Z}\n"


class Command(BaseCommand):
    """Show some information on invalid/bad urls"""

    def handle(self, *args, **kwargs):
        agency_list = Agency.objects.all()
        self.stdout.write(u"# Invalid URL Report\n")
        report_date = REPORT_DATE_FORMATTER.format(timezone.localtime(timezone.now()))
        self.stdout.write(u"Report generated: {0}\n\n".format(report_date))
        for agency in agency_list:
            self.stdout.write('## Agency: {0}\n\n'.format(agency.name))
            probe_list = Probe.objects.filter(audit__agency=agency, result__contains={'valid_url': 'false'})
            if probe_list.count() == 0:
                self.stdout.write('None!\n\n')
            for probe in probe_list:
                self.stdout.write('* {0}'.format(probe.result.get('initial_url', '???')))
            if probe_list.count() > 0:
                self.stdout.write('\n')

<commit_msg>Add url count per agency to the invalid url report<commit_after>from __future__ import division
from django.core.management.base import BaseCommand
from django.utils import timezone
from thezombies.models import (Agency, Probe)

REPORT_DATE_FORMATTER = u"{:%Y-%m-%d %I:%M%p %Z}\n"


class Command(BaseCommand):
    """Show some information on invalid/bad urls"""

    def handle(self, *args, **kwargs):
        agency_list = Agency.objects.all()
        self.stdout.write(u"# Invalid URL Report\n")
        report_date = REPORT_DATE_FORMATTER.format(timezone.localtime(timezone.now()))
        self.stdout.write(u"Report generated: {0}\n\n".format(report_date))
        for agency in agency_list:
            self.stdout.write('## Agency: {0}\n\n'.format(agency.name))
            probe_list = Probe.objects.filter(audit__agency=agency, result__contains={'valid_url': 'false'})
            if probe_list.count() == 0:
                self.stdout.write('None!\n\n')
            else:
                self.stdout.write('URL Count: {0}\n\n'.format(probe_list.count()))
            for probe in probe_list:
                self.stdout.write('* {0}'.format(probe.result.get('initial_url', '???')))
            if probe_list.count() > 0:
                self.stdout.write('\n')
{'fraction_non_alphanumeric': 0.14713216957605985, 'fraction_numerical': 0.00457190357439734, 'mean_word_length': 2.9588815789473686, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5522532', 'n_tokens_mistral': 757, 'n_tokens_neox': 739, 'n_words': 151}