<commit_before>from django.db.models import Q
from django.core.management.base import BaseCommand
from calaccess_raw.models import FilernameCd, FilerFilingsCd
from calaccess_raw.models import FilerLinksCd, SmryCd
from calaccess_campaign_browser.models import Committee, Filer


class Command(BaseCommand):

    def handle(self, *args, **options):
        """
        Take a look in the Filings table and just
        load up the filers that have filed a 460 or a 450.
        Load the candidates first,
        linking them to all the committees they control.
        Then load the rest of the committees.
        """
        candidate_cmte_list = []
        # all filers of type candidate
        all_candidate_filer_ids = (
            FilernameCd
            .objects
            .filter(filer_type='CANDIDATE/OFFICEHOLDER')
            .values_list('filer_id', flat=True)
            .distinct()
        )

        for filer_id in all_candidate_filer_ids:
            # querying by filer_id_a gets list of
            # connected recipient committees
            qs_linked = (
                FilerLinksCd
                .objects
                .filter(Q(filer_id_a=filer_id) | Q(filer_id_b=filer_id))
                .filter(link_type='12011')
            )

            filer_a_ids = list(
                (
                    qs_linked
                    .values_list('filer_id_a', flat=True)
                    .exclude(filer_id_a=filer_id)
                )
            )

            filer_b_ids = list(
                (
                    qs_linked
                    .values_list('filer_id_b', flat=True)
                    .exclude(filer_id_b=filer_id)
                )
            )

            cmte_filer_ids = filer_a_ids + filer_b_ids

            if len(cmte_filer_ids) > 0:  # has a linked committee
                # build a list of all candidate committees so can exclude for
                # pac import
                candidate_cmte_list.extend(cmte_filer_ids)
                # query to see if there are any filings associated with the
                # committee
                qs_cmte_filings = FilerFilingsCd.objects.filter(
                    filer_id__in=cmte_filer_ids)
                # has data associated with a committee
                if qs_cmte_filings.count() > 0:
                    candidate_filer_name_obj = (
                        FilernameCd
                        .objects
                        .filter(filer_id=filer_id)
                        .order_by('-effect_dt')
                        .exclude(naml='')[0]
                    )

                    candidate_filer_obj, created = Filer.objects.get_or_create(
                        filer_id=filer_id,
                        status=candidate_filer_name_obj.status,
                        filer_type='cand',
                        effective_date=candidate_filer_name_obj.effect_dt,
                        xref_filer_id=candidate_filer_name_obj.xref_filer_id,
                        name=(
                            '{0} {1} {2} {3}'
                            .format(
                                candidate_filer_name_obj.namt,
                                candidate_filer_name_obj.namf,
                                candidate_filer_name_obj.naml,
                                candidate_filer_name_obj.nams
                            )
                            .strip()
                        )
                    )

                    if created:
                        for cmte_filer_id in cmte_filer_ids:
                            qs_this_cmte_filings = (
                                qs_cmte_filings
                                .filter(filer_id=cmte_filer_id)
                                .values_list('filing_id', flat=True)
                            )

                            # filings have data
                            summary_count = (
                                SmryCd
                                .objects
                                .filter(
                                    filing_id__in=(
                                        list(qs_this_cmte_filings)
                                    )
                                )
                                .count()
                            )

                            if summary_count > 0:
                                candidate_filer_obj.active = True
                                candidate_filer_obj.save()

                                qs_names = (
                                    FilernameCd
                                    .objects
                                    .filter(filer_id=cmte_filer_id)
                                    .order_by('-effect_dt')
                                    .exclude(naml='')
                                )

                                if qs_names.count() > 0:
                                    cmte_filer_name_obj = qs_names[0]

                                    cmte_obj, cmte_created = (
                                        Committee
                                        .objects
                                        .get_or_create(
                                            filer=candidate_filer_obj,
                                            filer_id_raw=cmte_filer_id,
                                            name=(
                                                '{0} {1} {2} {3}'
                                                .format(
                                                    cmte_filer_name_obj.namt,
                                                    cmte_filer_name_obj.namf,
                                                    cmte_filer_name_obj.naml,
                                                    cmte_filer_name_obj.nams
                                                )
                                                .strip()
                                            ),
                                            committee_type='cand',
                                        )
                                    )

        print 'candidates and their linked committees loaded'

        all_filings = (
            FilerFilingsCd
            .objects
            .filter(Q(form_id='F460') | Q(form_id='F450'))
            .values_list('filing_id', flat=True)
            .distinct()
        )

        all_filings_with_data = (
            SmryCd
            .objects
            .filter(filing_id__in=all_filings)
            .values_list('filing_id', flat=True)
            .distinct()
        )

        # if you swap filing_id for filer_id in the values clause you
        # get the same count of filings as in all_filings_with_data
        all_filers_with_data = (
            FilerFilingsCd
            .objects
            .filter(filing_id__in=all_filings_with_data)
            .exclude(filer_id__in=candidate_cmte_list)
            .values_list('filer_id', flat=True)
            .distinct()
        )

        for pac_filer_id in all_filers_with_data:
            pac_filer_name = FilernameCd.objects.filter(filer_id=pac_filer_id)

            if pac_filer_name.count() > 0:
                pac_filer_name_obj = pac_filer_name.order_by(
                    '-effect_dt').exclude(naml='')[0]

                pac_filer_obj, created = Filer.objects.get_or_create(
                    filer_id=pac_filer_id,
                    status=pac_filer_name_obj.status,
                    filer_type='pac',
                    effective_date=pac_filer_name_obj.effect_dt,
                    xref_filer_id=pac_filer_name_obj.xref_filer_id,
                    name=(
                        '{0} {1} {2} {3}'
                        .format(
                            pac_filer_name_obj.namt,
                            pac_filer_name_obj.namf,
                            pac_filer_name_obj.naml,
                            pac_filer_name_obj.nams
                        )
                        .strip()
                    ),
                )
                qs_pac_filings = FilerFilingsCd.objects.filter(
                    filer_id=pac_filer_id)

                if qs_pac_filings.count() > 0:
                    qs_pac_smry = (
                        SmryCd
                        .objects
                        .filter(
                            filing_id__in=(
                                list(
                                    qs_pac_filings
                                    .values_list('filing_id', flat=True)
                                )
                            )
                        )
                    )

                    if qs_pac_smry.count() > 0:
                        pac_filer_obj.active = True
                        pac_filer_obj.save()

                pac_cmte_obj, pac_created = Committee.objects.get_or_create(
                    filer=pac_filer_obj,
                    filer_id_raw=pac_filer_id,
                    name=(
                        '{0} {1} {2} {3}'
                        .format(
                            pac_filer_name_obj.namt,
                            pac_filer_name_obj.namf,
                            pac_filer_name_obj.naml,
                            pac_filer_name_obj.nams
                        )
                        .strip()
                    ),
                    committee_type = 'pac',
                )

        print 'loaded non-candidate linked committees with associated filings'
    # Need to fix duped committee issue and deal with controlling filer thing.
<commit_msg>Imaginary restructure for mgmt command.<commit_after>from django.db.models import Q
from django.core.management.base import BaseCommand
from calaccess_raw.models import FilernameCd, FilerFilingsCd
from calaccess_raw.models import FilerLinksCd, SmryCd
from calaccess_campaign_browser.models import Committee, Filer


class Command(BaseCommand):

    def handle(self, *args, **options):
        # Imaginary restructure
        self.load_candidates()
        self.load_committees()
        self.link_committees_to_candidates()

    def oldschool(self, *args, **kwargs):
        """
        Take a look in the Filings table and just
        load up the filers that have filed a 460 or a 450.
        Load the candidates first,
        linking them to all the committees they control.
        Then load the rest of the committees.
        """
        candidate_cmte_list = []
        # all filers of type candidate
        all_candidate_filer_ids = (
            FilernameCd
            .objects
            .filter(filer_type='CANDIDATE/OFFICEHOLDER')
            .values_list('filer_id', flat=True)
            .distinct()
        )

        for filer_id in all_candidate_filer_ids:
            # querying by filer_id_a gets list of
            # connected recipient committees
            qs_linked = (
                FilerLinksCd
                .objects
                .filter(Q(filer_id_a=filer_id) | Q(filer_id_b=filer_id))
                .filter(link_type='12011')
            )

            filer_a_ids = list(
                (
                    qs_linked
                    .values_list('filer_id_a', flat=True)
                    .exclude(filer_id_a=filer_id)
                )
            )

            filer_b_ids = list(
                (
                    qs_linked
                    .values_list('filer_id_b', flat=True)
                    .exclude(filer_id_b=filer_id)
                )
            )

            cmte_filer_ids = filer_a_ids + filer_b_ids

            if len(cmte_filer_ids) > 0:  # has a linked committee
                # build a list of all candidate committees so can exclude for
                # pac import
                candidate_cmte_list.extend(cmte_filer_ids)
                # query to see if there are any filings associated with the
                # committee
                qs_cmte_filings = FilerFilingsCd.objects.filter(
                    filer_id__in=cmte_filer_ids)
                # has data associated with a committee
                if qs_cmte_filings.count() > 0:
                    candidate_filer_name_obj = (
                        FilernameCd
                        .objects
                        .filter(filer_id=filer_id)
                        .order_by('-effect_dt')
                        .exclude(naml='')[0]
                    )

                    candidate_filer_obj, created = Filer.objects.get_or_create(
                        filer_id=filer_id,
                        status=candidate_filer_name_obj.status,
                        filer_type='cand',
                        effective_date=candidate_filer_name_obj.effect_dt,
                        xref_filer_id=candidate_filer_name_obj.xref_filer_id,
                        name=(
                            '{0} {1} {2} {3}'
                            .format(
                                candidate_filer_name_obj.namt,
                                candidate_filer_name_obj.namf,
                                candidate_filer_name_obj.naml,
                                candidate_filer_name_obj.nams
                            )
                            .strip()
                        )
                    )

                    if created:
                        for cmte_filer_id in cmte_filer_ids:
                            qs_this_cmte_filings = (
                                qs_cmte_filings
                                .filter(filer_id=cmte_filer_id)
                                .values_list('filing_id', flat=True)
                            )

                            # filings have data
                            summary_count = (
                                SmryCd
                                .objects
                                .filter(
                                    filing_id__in=(
                                        list(qs_this_cmte_filings)
                                    )
                                )
                                .count()
                            )

                            if summary_count > 0:
                                candidate_filer_obj.active = True
                                candidate_filer_obj.save()

                                qs_names = (
                                    FilernameCd
                                    .objects
                                    .filter(filer_id=cmte_filer_id)
                                    .order_by('-effect_dt')
                                    .exclude(naml='')
                                )

                                if qs_names.count() > 0:
                                    cmte_filer_name_obj = qs_names[0]

                                    cmte_obj, cmte_created = (
                                        Committee
                                        .objects
                                        .get_or_create(
                                            filer=candidate_filer_obj,
                                            filer_id_raw=cmte_filer_id,
                                            name=(
                                                '{0} {1} {2} {3}'
                                                .format(
                                                    cmte_filer_name_obj.namt,
                                                    cmte_filer_name_obj.namf,
                                                    cmte_filer_name_obj.naml,
                                                    cmte_filer_name_obj.nams
                                                )
                                                .strip()
                                            ),
                                            committee_type='cand',
                                        )
                                    )

        print 'candidates and their linked committees loaded'

        all_filings = (
            FilerFilingsCd
            .objects
            .filter(Q(form_id='F460') | Q(form_id='F450'))
            .values_list('filing_id', flat=True)
            .distinct()
        )

        all_filings_with_data = (
            SmryCd
            .objects
            .filter(filing_id__in=all_filings)
            .values_list('filing_id', flat=True)
            .distinct()
        )

        # if you swap filing_id for filer_id in the values clause you
        # get the same count of filings as in all_filings_with_data
        all_filers_with_data = (
            FilerFilingsCd
            .objects
            .filter(filing_id__in=all_filings_with_data)
            .exclude(filer_id__in=candidate_cmte_list)
            .values_list('filer_id', flat=True)
            .distinct()
        )

        for pac_filer_id in all_filers_with_data:
            pac_filer_name = FilernameCd.objects.filter(filer_id=pac_filer_id)

            if pac_filer_name.count() > 0:
                pac_filer_name_obj = pac_filer_name.order_by(
                    '-effect_dt').exclude(naml='')[0]

                pac_filer_obj, created = Filer.objects.get_or_create(
                    filer_id=pac_filer_id,
                    status=pac_filer_name_obj.status,
                    filer_type='pac',
                    effective_date=pac_filer_name_obj.effect_dt,
                    xref_filer_id=pac_filer_name_obj.xref_filer_id,
                    name=(
                        '{0} {1} {2} {3}'
                        .format(
                            pac_filer_name_obj.namt,
                            pac_filer_name_obj.namf,
                            pac_filer_name_obj.naml,
                            pac_filer_name_obj.nams
                        )
                        .strip()
                    ),
                )
                qs_pac_filings = FilerFilingsCd.objects.filter(
                    filer_id=pac_filer_id)

                if qs_pac_filings.count() > 0:
                    qs_pac_smry = (
                        SmryCd
                        .objects
                        .filter(
                            filing_id__in=(
                                list(
                                    qs_pac_filings
                                    .values_list('filing_id', flat=True)
                                )
                            )
                        )
                    )

                    if qs_pac_smry.count() > 0:
                        pac_filer_obj.active = True
                        pac_filer_obj.save()

                pac_cmte_obj, pac_created = Committee.objects.get_or_create(
                    filer=pac_filer_obj,
                    filer_id_raw=pac_filer_id,
                    name=(
                        '{0} {1} {2} {3}'
                        .format(
                            pac_filer_name_obj.namt,
                            pac_filer_name_obj.namf,
                            pac_filer_name_obj.naml,
                            pac_filer_name_obj.nams
                        )
                        .strip()
                    ),
                    committee_type = 'pac',
                )

        print 'loaded non-candidate linked committees with associated filings'
    # Need to fix duped committee issue and deal with controlling filer thing.

{'fraction_non_alphanumeric': 0.05800763122615242, 'fraction_numerical': 0.004434361142621429, 'mean_word_length': 0.8464394516374715, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 88, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6348781', 'n_tokens_mistral': 4934, 'n_tokens_neox': 4668, 'n_words': 779}