<commit_before>from django.db.models import Manager
from django.db.models import F
from django.db.models import Q
from django.apps import apps
import django_rq


class HumanManager(Manager):
    def update_persons(self, cursor=None):
        # Get base
        humans = self.all()
        # Filter if cursored
        if cursor:
            humans = humans.filter(
                modified__gt=cursor,
            )
        # Return as objects
        humans = humans.values_list(
            'id',
            'first_name',
            'middle_name',
            'last_name',
            'nick_name',
            'email',
            'birth_date',
            'phone',
            'cell_phone',
            'work_phone',
            'bhs_id',
            'sex',
            'primary_voice_part',
            'is_deceased',
        )

        # Creating/Update Persons
        Person = apps.get_model('api.person')
        for human in humans:
            django_rq.enqueue(
                Person.objects.update_or_create_from_human,
                human,
                is_object=True,
            )
        t = humans.count()
        return t

    def delete_orphans(self):
        # Get base
        humans = self.all()
        humans = list(humans.values_list('id', flat=True))
        # Delete Orphans
        Person = apps.get_model('api.person')
        orphans = Person.objects.filter(
            mc_pk__isnull=False,
        ).exclude(
            mc_pk__in=humans,
        )
        t = orphans.count()
        orphans.delete()
        return t


class StructureManager(Manager):
    def update_groups(self, cursor=None):
        # Get base
        structures = self.all()
        # Filter if cursored
        if cursor:
            structures = structures.filter(
                modified__gt=cursor,
            )
        # Return as objects
        structures = structures.select_related(
            'status',
        ).values_list(
            'id',
            'name',
            'preferred_name',
            'chorus_name',
            'status__name',
            'kind',
            'established_date',
            'email',
            'phone',
            'website',
            'facebook',
            'twitter',
            'bhs_id',
            'parent__id',
            'chapter_code',
        )
        # Creating/Update Groups
        Group = apps.get_model('api.group')
        for structure in structures:
            django_rq.enqueue(
                Group.objects.update_or_create_from_structure,
                structure,
                is_object=True,
            )
        return structures.count()

    def delete_orphans(self):
        # Get base
        structures = self.all()
        structures = list(structures.values_list('id', flat=True))
        # Delete Orphans
        Group = apps.get_model('api.group')
        orphans = Group.objects.filter(
            mc_pk__isnull=False,
        ).exclude(
            mc_pk__in=structures,
        )
        t = orphans.count()
        orphans.delete()
        return t


class RoleManager(Manager):
    def update_officers(self, cursor=None):
        # Get the cursor
        Officer = apps.get_model('api.officer')

        # Get base
        roles = self.select_related(
            'structure',
            'human',
        )
        # Will rebuild without a cursor
        if cursor:
            roles = roles.filter(
                modified__gt=cursor,
            )
        # Order and Return as objects
        roles = roles.order_by(
            'modified'
        ).values_list(
            'id',
            'name',
            'structure',
            'human',
            'start_date',
            'end_date',
        )

        # Creating/Update Officers
        Officer = apps.get_model('api.officer')
        for role in roles:
            django_rq.enqueue(
                Officer.objects.update_from_role,
                role,
                is_object=True,
            )
        return roles.count()


class JoinManager(Manager):
    def update_members(self, cursor=None):
        Member = apps.get_model('api.member')
        # Get base
        joins = self.select_related(
            'structure',
            'subscription',
            'subscription__human',
            'membership',
            'membership__status',
        )
        if cursor:
            joins = joins.filter(
                modified__gt=cursor,
            )
        # Order and Return as objects
        joins = joins.order_by(
            F('modified').asc(nulls_first=True),
            'created',
        ).values_list(
            'id',
            'structure__id',
            'subscription__human__id',
            'inactive_date',
            'inactive_reason',<commit_msg>UPdate logic to match modified TS<commit_after>from django.db.models import Manager
from django.apps import apps
import django_rq


class HumanManager(Manager):
    def update_persons(self, cursor=None):
        # Get base
        humans = self.all()
        # Filter if cursored
        if cursor:
            humans = humans.filter(
                modified__gt=cursor,
            )
        # Return as objects
        humans = humans.values_list(
            'id',
            'first_name',
            'middle_name',
            'last_name',
            'nick_name',
            'email',
            'birth_date',
            'phone',
            'cell_phone',
            'work_phone',
            'bhs_id',
            'sex',
            'primary_voice_part',
            'is_deceased',
        )

        # Creating/Update Persons
        Person = apps.get_model('api.person')
        for human in humans:
            django_rq.enqueue(
                Person.objects.update_or_create_from_human,
                human,
                is_object=True,
            )
        t = humans.count()
        return t

    def delete_orphans(self):
        # Get base
        humans = self.all()
        humans = list(humans.values_list('id', flat=True))
        # Delete Orphans
        Person = apps.get_model('api.person')
        orphans = Person.objects.filter(
            mc_pk__isnull=False,
        ).exclude(
            mc_pk__in=humans,
        )
        t = orphans.count()
        orphans.delete()
        return t


class StructureManager(Manager):
    def update_groups(self, cursor=None):
        # Get base
        structures = self.all()
        # Filter if cursored
        if cursor:
            structures = structures.filter(
                modified__gt=cursor,
            )
        # Return as objects
        structures = structures.select_related(
            'status',
        ).values_list(
            'id',
            'name',
            'preferred_name',
            'chorus_name',
            'status__name',
            'kind',
            'established_date',
            'email',
            'phone',
            'website',
            'facebook',
            'twitter',
            'bhs_id',
            'parent__id',
            'chapter_code',
        )
        # Creating/Update Groups
        Group = apps.get_model('api.group')
        for structure in structures:
            django_rq.enqueue(
                Group.objects.update_or_create_from_structure,
                structure,
                is_object=True,
            )
        return structures.count()

    def delete_orphans(self):
        # Get base
        structures = self.all()
        structures = list(structures.values_list('id', flat=True))
        # Delete Orphans
        Group = apps.get_model('api.group')
        orphans = Group.objects.filter(
            mc_pk__isnull=False,
        ).exclude(
            mc_pk__in=structures,
        )
        t = orphans.count()
        orphans.delete()
        return t


class RoleManager(Manager):
    def update_officers(self, cursor=None):
        # Get the cursor
        Officer = apps.get_model('api.officer')

        # Get base
        roles = self.select_related(
            'structure',
            'human',
        )
        # Will rebuild without a cursor
        if cursor:
            roles = roles.filter(
                modified__gt=cursor,
            )
        # Order and Return as objects
        roles = roles.order_by(
            'modified'
        ).values_list(
            'id',
            'name',
            'structure',
            'human',
            'start_date',
            'end_date',
        )

        # Creating/Update Officers
        Officer = apps.get_model('api.officer')
        for role in roles:
            django_rq.enqueue(
                Officer.objects.update_from_role,
                role,
                is_object=True,
            )
        return roles.count()


class JoinManager(Manager):
    def update_members(self, cursor=None):
        Member = apps.get_model('api.member')
        # Get base
        joins = self.select_related(
            'structure',
            'subscription',
            'subscription__human',
            'membership',
            'membership__status',
        )
        if cursor:
            joins = joins.filter(
                modified__gt=cursor,
            )
        # Order and Return as objects
        joins = joins.order_by(
            'modified',
        ).values_list(
            'id',
            'structure__id',
            'subscription__human__id',
            'inactive_date',
            'inactive_reason',
{'fraction_non_alphanumeric': 0.09397463002114165, 'fraction_numerical': 0.0, 'mean_word_length': 1.3640679660169914, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6165423', 'n_tokens_mistral': 2585, 'n_tokens_neox': 2530, 'n_words': 515}