<commit_before>from django.core.management.base import BaseCommand
from hmda.models import HMDARecord

class Command(BaseCommand):
    args = "None"
    help = """ Validate HMDA Data in FairLending DB is Correct"""

    def handle(self, *args, **options):
        print "Begin Validation"
        '''
        #  To test this go to https://api.consumerfinance.gov/data/hmda/slice/hmda_lar.html?
        #  filter by where clasue:  as_of_year=2013
        #                           and census_tract_number != ''
        #                           and state_code=<insert_state_num_here>
        '''
        states_with_Q_Counts = {"12":944260,
                                "17":635804,
                                "18":341612,
                                "55":303365
                               }

        for key,value in states_with_Q_Counts.items():
            db_count = HMDARecord.objects.filter(statefp=key).count()
            print "Testing State " + key
            assert self.test_records(db_count,value)

        print "Congrats! All HMDA Record Counts Passed"


    def test_records(self,db_count, Q_count):
        if db_count == Q_count:
            print "Passed"
            return True

        print "Failed " + str(db_count), str(Q_count)
        return False<commit_msg>added new states<commit_after>from django.core.management.base import BaseCommand
from hmda.models import HMDARecord

class Command(BaseCommand):
    args = "None"
    help = """ Validate HMDA Data in FairLending DB is Correct"""

    def handle(self, *args, **options):
        print "Begin Validation"
        '''
        #  To test this go to https://api.consumerfinance.gov/data/hmda/slice/hmda_lar.html?
        #  filter by where clasue:  as_of_year=2013
        #                           and census_tract_number != ''
        #                           and state_code=<insert_state_num_here>
        '''
        states_with_Q_Counts = {"12":944260,
                                "17":635804,
                                "18":341612,
                                "55":303365,
                                "06":2158015,
                                "13":534125,
                               }

        for key,value in states_with_Q_Counts.items():
            db_count = HMDARecord.objects.filter(statefp=key).count()
            print "Testing State " + key
            assert self.test_records(db_count,value)

        print "Congrats! All HMDA Record Counts Passed"


    def test_records(self,db_count, Q_count):
        if db_count == Q_count:
            print "Passed"
            return True

        print "Failed " + str(db_count), str(Q_count)
        return False
{'fraction_non_alphanumeric': 0.09198374584410787, 'fraction_numerical': 0.0328777244181751, 'mean_word_length': 1.5913875598086125, 'pattern_counts': {'":': 10, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3102943', 'n_tokens_mistral': 839, 'n_tokens_neox': 758, 'n_words': 169}