<commit_before>import gc

from django.core.management.base import BaseCommand
from django.db import reset_queries
from django.db.models import Q

try:
    from calaccess_raw.models import (
        FilernameCd,
        FilerFilingsCd,
        FilerLinksCd,
        ExpnCd, RcptCd,
        SmryCd
    )

except:
    print 'you need to load the raw calaccess data app'

from campaign_finance.models import (
    Committee,
    Contribution,
    Cycle,
    Expenditure,
    Filer,
    Filing,
    Summary
)


def queryset_iterator(queryset, chunksize=1000):
    '''
    Iterate over a Django Queryset ordered by the primary key

    This method loads a maximum of chunksize (default: 1000) rows in it's
    memory at the same time while django normally would load all rows in it's
    memory. Using the iterator() method only causes it to not preload all the
    classes.

    Note that the implementation of the iterator
    does not support ordered query sets.

    https://djangosnippets.org/snippets/1949/
    '''
    pk = 0
    last_pk = queryset.order_by('-pk')[0].pk
    queryset = queryset.order_by('pk')
    while pk < last_pk:
        for row in queryset.filter(pk__gt=pk)[:chunksize]:
            pk = row.pk
            yield row
        gc.collect()


def insert_cmte(filer_obj, cmte_filer_id_raw, cmte_type,
                cmte_name, cmte_xref_id, effective_date):

    insert_cmtee = Committee()
    # tie all candidate committees to the CAL-ACCESS candidate filer
    insert_cmtee.filer = filer_obj
    # preserve the CAL-ACCESS filer_id for the committee here, but don't add
    # it to our Filer model. So can distinguish PAC from Cand Cmtee
    insert_cmtee.filer_id_raw = cmte_filer_id_raw
    insert_cmtee.committee_type = cmte_type
    insert_cmtee.name = cmte_name
    insert_cmtee.xref_filer_id = cmte_xref_id
    insert_cmtee.save()


class Command(BaseCommand):
    help = 'Isolate recipient committee campaign finance data'

    def handle(self, *args, **options):
        # self.load_filers()
        # self.load_filings()
        # self.load_summary()
        # self.load_contributions()
        self.load_expenditures()

    def load_filers(self):
        '''
        Take a look in the Filings table and just
        load up the filers that have filed a 460 or a 450.
        Load the candidates first,
        linking them to all the committees they control.
        Then load the rest of the committees.
        '''
        candidate_cmte_list = []
        # all filers of type candidate
        all_candidate_filer_ids = (
            FilernameCd
            .objects
            .filter(filer_type='CANDIDATE/OFFICEHOLDER')
            .values_list('filer_id', flat=True)
            .distinct()
        )

        for filer_id in all_candidate_filer_ids:
            # querying by filer_id_a gets list of<commit_msg>Uncomment other parts of the big loader<commit_after>import gc

from django.core.management.base import BaseCommand
from django.db import reset_queries
from django.db.models import Q

try:
    from calaccess_raw.models import (
        FilernameCd,
        FilerFilingsCd,
        FilerLinksCd,
        ExpnCd, RcptCd,
        SmryCd
    )

except:
    print 'you need to load the raw calaccess data app'

from campaign_finance.models import (
    Committee,
    Contribution,
    Cycle,
    Expenditure,
    Filer,
    Filing,
    Summary
)


def queryset_iterator(queryset, chunksize=1000):
    '''
    Iterate over a Django Queryset ordered by the primary key

    This method loads a maximum of chunksize (default: 1000) rows in it's
    memory at the same time while django normally would load all rows in it's
    memory. Using the iterator() method only causes it to not preload all the
    classes.

    Note that the implementation of the iterator
    does not support ordered query sets.

    https://djangosnippets.org/snippets/1949/
    '''
    pk = 0
    last_pk = queryset.order_by('-pk')[0].pk
    queryset = queryset.order_by('pk')
    while pk < last_pk:
        for row in queryset.filter(pk__gt=pk)[:chunksize]:
            pk = row.pk
            yield row
        gc.collect()


def insert_cmte(filer_obj, cmte_filer_id_raw, cmte_type,
                cmte_name, cmte_xref_id, effective_date):

    insert_cmtee = Committee()
    # tie all candidate committees to the CAL-ACCESS candidate filer
    insert_cmtee.filer = filer_obj
    # preserve the CAL-ACCESS filer_id for the committee here, but don't add
    # it to our Filer model. So can distinguish PAC from Cand Cmtee
    insert_cmtee.filer_id_raw = cmte_filer_id_raw
    insert_cmtee.committee_type = cmte_type
    insert_cmtee.name = cmte_name
    insert_cmtee.xref_filer_id = cmte_xref_id
    insert_cmtee.save()


class Command(BaseCommand):
    help = 'Isolate recipient committee campaign finance data'

    def handle(self, *args, **options):
        self.load_filers()
        self.load_filings()
        self.load_summary()
        self.load_contributions()
        self.load_expenditures()

    def load_filers(self):
        '''
        Take a look in the Filings table and just
        load up the filers that have filed a 460 or a 450.
        Load the candidates first,
        linking them to all the committees they control.
        Then load the rest of the committees.
        '''
        candidate_cmte_list = []
        # all filers of type candidate
        all_candidate_filer_ids = (
            FilernameCd
            .objects
            .filter(filer_type='CANDIDATE/OFFICEHOLDER')
            .values_list('filer_id', flat=True)
            .distinct()
        )

        for filer_id in all_candidate_filer_ids:
            # querying by filer_id_a gets list of
{'fraction_non_alphanumeric': 0.06750612102133613, 'fraction_numerical': 0.006995452955578874, 'mean_word_length': 2.510742786985881, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2767807', 'n_tokens_mistral': 1824, 'n_tokens_neox': 1735, 'n_words': 535}