<commit_before># -*- coding: UTF-8 -*-
from __future__ import unicode_literals

from optparse import make_option

from django.core.management.base import BaseCommand
from django.db.models import Model, Q

from tx_highered.models import Institution


class Command(BaseCommand):
    help = (
        'Create a report card of our data quality. Run it through '
        'http://setosa.io/csv-fingerprint/ for a quick visualization.'
    )
    option_list = BaseCommand.option_list + (
        make_option('--year',
            dest='year',
            type='int',
            help='Data must be from at least this year'),
    )

    def handle(self, *args, **options):

        def latest_year(model_or_queryset, relation=None):
            """Latest year institution has data, returns year or ''."""
            if relation is not None and isinstance(model_or_queryset, Model):
                queryset = getattr(model_or_queryset, relation).all()
            else:
                queryset = model_or_queryset
            try:
                year = queryset.order_by('-year')[0].year
                if options['year'] is not None and year < options['year']:
                    return ''
                else:
                    return year
            except IndexError:
                return ''

        header = (
            'slug',
            'Admissions IPEDS',
            'Admissions THECB',
            'Test Scores',
            'Full-Time Equivalent',
            'Demographics IPEDS',
            'Demographics THECB',
            'Price Trends',
            'Graduation IPEDS',
            'Graduation THECB',
        )
        print ','.join(header)
        institutions = (Institution.objects.published()
            .order_by('institution_type', 'is_private', 'name'))
        for institution in institutions:
            row = [institution.slug]
            row.append(latest_year(institution, 'admissions'))
            if institution.is_private:
                row.append('NA')
            else:
                row.append(latest_year(institution, 'publicadmissions'))<commit_msg>add the institution type to the report card so I can prioritize<commit_after># -*- coding: UTF-8 -*-
from __future__ import unicode_literals

from optparse import make_option

from django.core.management.base import BaseCommand
from django.db.models import Model, Q

from tx_highered.models import Institution


class Command(BaseCommand):
    help = (
        'Create a report card of our data quality. Run it through '
        'http://setosa.io/csv-fingerprint/ for a quick visualization.'
    )
    option_list = BaseCommand.option_list + (
        make_option('--year',
            dest='year',
            type='int',
            help='Data must be from at least this year'),
    )

    def handle(self, *args, **options):

        def latest_year(model_or_queryset, relation=None):
            """Latest year institution has data, returns year or ''."""
            if relation is not None and isinstance(model_or_queryset, Model):
                queryset = getattr(model_or_queryset, relation).all()
            else:
                queryset = model_or_queryset
            try:
                year = queryset.order_by('-year')[0].year
                if options['year'] is not None and year < options['year']:
                    return ''
                else:
                    return year
            except IndexError:
                return ''

        header = (
            'slug',
            'type',
            'Admissions IPEDS',
            'Admissions THECB',
            'Test Scores',
            'Full-Time Equivalent',
            'Demographics IPEDS',
            'Demographics THECB',
            'Price Trends',
            'Graduation IPEDS',
            'Graduation THECB',
        )
        print ','.join(header)
        institutions = (Institution.objects.published()
            .order_by('institution_type', 'is_private', 'name'))
        for institution in institutions:
            row = [institution.slug, institution.institution_type]
            row.append(latest_year(institution, 'admissions'))
            if institution.is_private:
                row.append('NA')
            else:
                row.append(latest_year(institution, 'publicadmissions'))
{'fraction_non_alphanumeric': 0.09958217270194986, 'fraction_numerical': 0.0009285051067780873, 'mean_word_length': 1.836734693877551, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3016681', 'n_tokens_mistral': 1155, 'n_tokens_neox': 1104, 'n_words': 300}