<commit_before>import pupa.scrape
from pupa.utils import make_psuedo_id

from legistar.utils.itemgenerator import make_item
from legistar.ext.pupa.base import Adapter, Converter, SetDefault


class MembershipAdapter(Adapter):
    '''Convert a legistar scraper's membership into a pupa-compliant
    membership.
    '''
    pupa_model = pupa.scrape.Membership
    extras_keys = ['appointed_by']

    def stringify_date(self, dt):
        '''Given a datetime string, stringify it to a date,
        assuming there is no time portion associated with the date.
        Complain if there is.
        '''
        if dt is None:
            raise self.SkipItem()
        else:
            return dt.strftime('%Y-%m-%d')

    @make_item('start_date')
    def get_start_date(self):
        return self.stringify_date(self.data.get('start_date'))

    @make_item('end_date')
    def get_end_date(self):
        return self.stringify_date(self.data.get('end_date'))

    @make_item('organization_id')
    def get_org_id(self):
        return self.data['organization_id']

    def get_instance(self, **extra_instance_data):
        # Get instance data.
        instance_data = self.get_instance_data()
        instance_data.update(extra_instance_data)
        extras = instance_data.pop('extras')

        # Create the instance.
        instance = self.pupa_model(**instance_data)
        instance.extras.update(extras)
        return instance


class MembershipConverter(Converter):
    adapter = MembershipAdapter

    def __iter__(self):
        yield from self.create_memberships()

    def get_org(self, org_name):
        '''Gets or creates the org with name equal to
        kwargs['name']. Caches the result.
        '''
        created = False
        with SetDefault(self, 'orgs', {}) as orgs:
            # Get the org.
            org = orgs.get(org_name)

            if org is not None:
                # Cache hit.
                return created, org

            # Create the org.
            classification = self.cfg.get_org_classification(org_name)
            org = pupa.scrape.Organization(
                name=org_name, classification=classification)
            created = True

            # Cache it.
            orgs[org_name] = org

            # Add a source to the org.
            for source in self.person.sources:
                if 'detail' in source['note']:
                    org.add_source(**source)

        return created, org

    def create_membership(self, data):
        '''Retrieves the matching committee and adds this person
        as a member of the committee.
        '''
        if 'person_id' not in data:
            data['person_id'] = self.person._id

        # Get or create the committee.
        if 'organization_id' not in data:
            org_name = data.pop('org')
            created, org = self.get_org(org_name)
            if created:
                yield org

        # Add the person and org ids.
            data['organization_id'] = org._id

        # Convert the membership to pupa object.
        adapter = self.make_child(self.adapter, data)
        membership = adapter.get_instance()
        yield membership

    def create_memberships(self):
        # Yield the memberships found in the person's detail table.
        for membership in self.memberships:
            yield from self.create_membership(membership)

        # Also, if the person has a party, emit a party membership.
        if not self.party:
            return

        post_kwargs = {}
        post_kwargs['organization__classification'] = 'legislature'
        if self.district:
            post_kwargs['label'] = self.district

        membership = dict(
            person_id=self.person._id,
            organization_id=make_psuedo_id(classification="legislature"),
            post_id=make_psuedo_id(**post_kwargs),
            role='member',
            start_date=self.person._start_date,
            end_date=self.person._end_date)
        yield self.create_membership(membership)



# ------------------------------------------------------------------------
# People
# ------------------------------------------------------------------------
class PeopleAdapter(Adapter):
    '''Converts legistar data into a pupa.scrape.Person instance.
    Note the make_item methods are popping values out the dict,
    because the associated keys aren't valid pupa.scrape.Person fields.
    '''
    pupa_model = pupa.scrape.Person
    aliases = [('fullname', 'name'),]
    extras_keys = ['firstname', 'lastname', 'notes']

    @make_item('links', wrapwith=list)
    def get_links(self):
        '''Move the website link into the pupa links attr,
        '''
        website_url = self.data.pop('website', None)
        if website_url is not None:
            yield dict(note='website', url=website_url)

    @make_item('contact_details', wrapwith=list)
    def gen_contacts(self):
        '''Move legistar's top-level email into contacts dict.
        '''
        for key in 'email', 'fax':
            email = self.data.pop(key, None)<commit_msg>Creates and yields person's membership in party<commit_after>import pupa.scrape
from pupa.utils import make_psuedo_id

from legistar.utils.itemgenerator import make_item
from legistar.ext.pupa.base import Adapter, Converter, SetDefault


class MembershipAdapter(Adapter):
    '''Convert a legistar scraper's membership into a pupa-compliant
    membership.
    '''
    pupa_model = pupa.scrape.Membership
    extras_keys = ['appointed_by']

    def stringify_date(self, dt):
        '''Given a datetime string, stringify it to a date,
        assuming there is no time portion associated with the date.
        Complain if there is.
        '''
        if dt is None:
            raise self.SkipItem()
        else:
            return dt.strftime('%Y-%m-%d')

    @make_item('start_date')
    def get_start_date(self):
        return self.stringify_date(self.data.get('start_date'))

    @make_item('end_date')
    def get_end_date(self):
        return self.stringify_date(self.data.get('end_date'))

    @make_item('organization_id')
    def get_org_id(self):
        return self.data['organization_id']

    def get_instance(self, **extra_instance_data):
        # Get instance data.
        instance_data = self.get_instance_data()
        instance_data.update(extra_instance_data)
        extras = instance_data.pop('extras')

        # Create the instance.
        instance = self.pupa_model(**instance_data)
        instance.extras.update(extras)
        return instance


class MembershipConverter(Converter):
    adapter = MembershipAdapter

    def __iter__(self):
        yield from self.create_memberships()

    def get_org(self, org_name):
        '''Gets or creates the org with name equal to
        kwargs['name']. Caches the result.
        '''
        created = False
        with SetDefault(self, 'orgs', {}) as orgs:
            # Get the org.
            org = orgs.get(org_name)

            if org is not None:
                # Cache hit.
                return created, org

            # Create the org.
            classification = self.cfg.get_org_classification(org_name)
            org = pupa.scrape.Organization(
                name=org_name, classification=classification)
            created = True

            # Cache it.
            orgs[org_name] = org

            # Add a source to the org.
            for source in self.person.sources:
                if 'detail' in source['note']:
                    org.add_source(**source)

        return created, org

    def create_membership(self, data):
        '''Retrieves the matching committee and adds this person
        as a member of the committee.
        '''
        if 'person_id' not in data:
            data['person_id'] = self.person._id

        # Get or create the committee.
        if 'organization_id' not in data:
            org_name = data.pop('org')
            created, org = self.get_org(org_name)
            if created:
                yield org

        # Add the person and org ids.
            data['organization_id'] = org._id

        # Convert the membership to pupa object.
        adapter = self.make_child(self.adapter, data)
        membership = adapter.get_instance()
        yield membership

    def create_memberships(self):
        # Yield the memberships found in the person's detail table.
        for membership in self.memberships:
            yield from self.create_membership(membership)

        # Also, if the person has a party, emit a party membership.
        if not self.party:
            return

        post_kwargs = {}
        post_kwargs['organization__classification'] = 'legislature'
        if self.district:
            post_kwargs['label'] = self.district

        membership = dict(
            person_id=self.person._id,
            organization_id=make_psuedo_id(classification="legislature"),
            post_id=make_psuedo_id(**post_kwargs),
            role='member',
            start_date=self.person._start_date,
            end_date=self.person._end_date)
        yield self.create_membership(membership)

        # create a party membership
        if self.party:
            membership = dict(
                person_id=self.person._id,
                organization_id=make_psuedo_id(
                    classification="party", name=self.party),
                role='member')
            yield self.create_membership(membership)


# ------------------------------------------------------------------------
# People
# ------------------------------------------------------------------------
class PeopleAdapter(Adapter):
    '''Converts legistar data into a pupa.scrape.Person instance.
    Note the make_item methods are popping values out the dict,
    because the associated keys aren't valid pupa.scrape.Person fields.
    '''
    pupa_model = pupa.scrape.Person
    aliases = [('fullname', 'name'),]
    extras_keys = ['firstname', 'lastname', 'notes']

    @make_item('links', wrapwith=list)
    def get_links(self):
        '''Move the website link into the pupa links attr,
        '''
        website_url = self.data.pop('website', None)
        if website_url is not None:
            yield dict(note='website', url=website_url)

    @make_item('contact_details', wrapwith=list)
    def gen_contacts(self):
        '''Move legistar's top-level email into contacts dict.
        '''
        for key in 'email', 'fax':
            email = self.data.pop(key, None)
{'fraction_non_alphanumeric': 0.11785408541805384, 'fraction_numerical': 0.0, 'mean_word_length': 2.039606822781151, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3799897', 'n_tokens_mistral': 2961, 'n_tokens_neox': 2819, 'n_words': 768}