<commit_before>import re
import inspect
import datetime

from legistar.base import Base, ChainedLookup
from legistar.views import LegistarScraper
from legistar.utils.itemgenerator import ItemGenerator, make_item
from legistar.utils.handythings import CachedAttr, DictSetDefault, SetDefault

import pupa.scrape
from pupa.utils import make_psuedo_id


class PupaExtBase(Base):
    '''This class's attrs are inherited state values. If they are defined in
    an object's parent chainmap, the object can get/set them. Get's will
    occur on ancestor chainmaps; gets happen in the instances own map
    and don't affect ancestors.
    '''
    # ------------------------------------------------------------------------
    # Names set on the generator and availabe to its children:
    # ------------------------------------------------------------------------

    # The main generator (the object invoked by pupa). Will be the
    # parent chainmap of all other PupaExtBase types used in the module.
    generator = ChainedLookup('generator')
    # Cache of orgs; maps org names to instances.
    orgs = ChainedLookup('orgs')

    # ------------------------------------------------------------------------
    # Names set on the person converter and availabe to its children:
    # ------------------------------------------------------------------------
    # The current person being imported and similar:
    person = ChainedLookup('person')
    sources = ChainedLookup('sources')
    memberships = ChainedLookup('memberships')
    party = ChainedLookup('party')
    district = ChainedLookup('district')


class Adapter(PupaExtBase, ItemGenerator):
    '''Base class responsible for mutating legistar dict
    into dict suitable for invoking pupa models.
    '''
    # The pupa_model the transormed pupa data will be used to invoke.
    pupa_model = None
    # List of (old_key, new_key) pairs used to rename fields.
    aliases = []
    # Values at these keys will be moved into the extras dict.
    extras_keys = []
    # Values at these keys get dropped.
    drop_keys = []

    def __init__(self, legistar_data, *args, **kwargs):
        self.data = legistar_data
        self.args = args
        self.kwargs = kwargs

    def get_instance_data(self, **extra_instance_data):
        '''Converts legistar dict output into data suitable for
        pupa.scrape pupa_model invocation. For example, datetimes need to
        be stringified. Based on class-level properties, dict items
        will be renamed, overwritten, or moved into `extras`. Mutates
        the legistar dict in place.
        '''
        data = dict(self.data)
        date.update(extra_instance_data)

        # Collect the make_item output.
        data.update(self)

        # Apply the aliases.
        for oldkey, newkey in self.aliases:
            self.data[newkey] = self.data.pop(oldkey)

        # Move non-pupa keys into the extras dict.
        with DictSetDefault(self.data, 'extras', {}) as extras:
            for key in self.extras_keys:
                value = self.data.pop(key, None)
                if value is not None:
                    extras[key] = value

        # Drop keys.
        for key in self.drop_keys:
            data.pop(key, None)<commit_msg>Since we're no longer mutating legistar data in palce, update the logic for moving non-pupa values into extra keys<commit_after>import re
import inspect
import datetime

from legistar.base import Base, ChainedLookup
from legistar.views import LegistarScraper
from legistar.utils.itemgenerator import ItemGenerator, make_item
from legistar.utils.handythings import CachedAttr, DictSetDefault, SetDefault

import pupa.scrape
from pupa.utils import make_psuedo_id


class PupaExtBase(Base):
    '''This class's attrs are inherited state values. If they are defined in
    an object's parent chainmap, the object can get/set them. Get's will
    occur on ancestor chainmaps; gets happen in the instances own map
    and don't affect ancestors.
    '''
    # ------------------------------------------------------------------------
    # Names set on the generator and availabe to its children:
    # ------------------------------------------------------------------------

    # The main generator (the object invoked by pupa). Will be the
    # parent chainmap of all other PupaExtBase types used in the module.
    generator = ChainedLookup('generator')
    # Cache of orgs; maps org names to instances.
    orgs = ChainedLookup('orgs')

    # ------------------------------------------------------------------------
    # Names set on the person converter and availabe to its children:
    # ------------------------------------------------------------------------
    # The current person being imported and similar:
    person = ChainedLookup('person')
    sources = ChainedLookup('sources')
    memberships = ChainedLookup('memberships')
    party = ChainedLookup('party')
    district = ChainedLookup('district')


class Adapter(PupaExtBase, ItemGenerator):
    '''Base class responsible for mutating legistar dict
    into dict suitable for invoking pupa models.
    '''
    # The pupa_model the transormed pupa data will be used to invoke.
    pupa_model = None
    # List of (old_key, new_key) pairs used to rename fields.
    aliases = []
    # Values at these keys will be moved into the extras dict.
    extras_keys = []
    # Values at these keys get dropped.
    drop_keys = []

    def __init__(self, legistar_data, *args, **kwargs):
        self.data = legistar_data
        self.args = args
        self.kwargs = kwargs

    def get_instance_data(self, **extra_instance_data):
        '''Converts legistar dict output into data suitable for
        pupa.scrape pupa_model invocation. For example, datetimes need to
        be stringified. Based on class-level properties, dict items
        will be renamed, overwritten, or moved into `extras`. Mutates
        the legistar dict in place.
        '''
        data = dict(self.data)
        date.update(extra_instance_data)

        # Collect the make_item output.
        data.update(self)

        # Apply the aliases.
        for oldkey, newkey in self.aliases:
            data[newkey] = data.pop(oldkey)

        # Move non-pupa keys into the extras dict.
        with DictSetDefault(data, 'extras', {}) as extras:
            for key in self.extras_keys:
                value = data.pop(key, None)
                if value is not None:
                    extras[key] = value

        # Drop keys.
        for key in self.drop_keys:
            data.pop(key, None)
{'fraction_non_alphanumeric': 0.15313401123083928, 'fraction_numerical': 0.0, 'mean_word_length': 2.022935779816514, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5184625', 'n_tokens_mistral': 1750, 'n_tokens_neox': 1666, 'n_words': 631}