<commit_before>import re
import json
import collections
from datetime import datetime

from legistar.forms import Form
from legistar.tables import Table, TableRow
from legistar.views import SearchView, DetailView
from legistar.fields import FieldAggregator, make_item, gen_items
from legistar.base import DictSetDefault, NoClobberDict


class EventFields(FieldAggregator):

    @make_item('location')
    def get_location(self):
        return self.get_field_text('location')

    @make_item('documents', wrapwith=list)
    def gen_documents(self):
        for key in self.get_config_value('PUPA_DOCUMENTS'):
            field = self.field_data.get(key)

            # This column isn't present on this legistar instance.
            if field is None:
                continue
            elif field.is_blank():
                continue
            elif field.get_url() is None:
                continue
            document = dict(
                name=field.get_text(),
                url=field.get_url(),
                mimetype=field.get_mimetype())
            yield document

    @make_item('participants', wrapwith=list)
    def gen_participants(self):
        participant_fields = self.get_config_value('PUPA_PARTICIPANTS')
        for entity_type, keys in participant_fields.items():
            for key in keys:
                cell = self.field_data[key]
                participant = dict(name=cell.text, type=entity_type)
                yield participant

    @make_item('sources', wrapwith=list)
    def gen_sources(self):
        grouped = collections.defaultdict(set)
        for note, url in self.chainmap['sources'].items():
            grouped[url].add(note)
        for url, notes in grouped.items():
            yield dict(url=url, note=', '.join(notes))


class SearchView(SearchView):
    PUPATYPE = 'events'
    VIEWTYPE = 'search'
    sources_note = 'Events search'


class SearchTableRow(TableRow, EventFields):
    KEY_PREFIX = 'EVT_TABLE'

    def get_detail_url(self):
        return self.get_field_url('details')

    @CachedAttr
    def ical_data(self):
        ical_url = self.get_ical_url()
        self.debug('%r is fetching ical data: %r', self, ical_url)
        resp = self.cfg.client.session.get(ical_url)
        with DictSetDefault(self.chainmap, 'sources', {}) as sources:
            sources['Event icalendar data (end date)'] = ical_url
        return resp.text

    def get_ical_url(self):
        return self.get_field_url('ical')

    @make_item('name')
    def get_name(self):
        '''The pupa name, or Legistar "topic" of the meeting.
        '''
        return self.get_field_text('topic')

    @make_item('end')
    def get_end(self):
        '''Get the event end date from the ical record.
        '''
        end_time = re.search(r'DTEND:([\dT]+)', self.ical_data).group(1)
        dt = datetime.strptime(end_time, r'%Y%m%dT%H%M%S')
        return dt

    @make_item('when')
    def get_when(self):
        '''Get the event start date from the ical record.
        '''
        end_time = re.search(r'DTSTART:([\dT]+)', self.ical_data).group(1)
        dt = datetime.strptime(end_time, r'%Y%m%dT%H%M%S')
        return dt
<commit_msg>CachedAttr was wreaking havoc during inspect.getmembers<commit_after>import re
import json
import collections
from datetime import datetime

from legistar.forms import Form
from legistar.tables import Table, TableRow
from legistar.views import SearchView, DetailView
from legistar.fields import FieldAggregator, make_item, gen_items
from legistar.base import DictSetDefault, NoClobberDict


class EventFields(FieldAggregator):

    @make_item('location')
    def get_location(self):
        return self.get_field_text('location')

    @make_item('documents', wrapwith=list)
    def gen_documents(self):
        for key in self.get_config_value('PUPA_DOCUMENTS'):
            field = self.field_data.get(key)

            # This column isn't present on this legistar instance.
            if field is None:
                continue
            elif field.is_blank():
                continue
            elif field.get_url() is None:
                continue
            document = dict(
                name=field.get_text(),
                url=field.get_url(),
                mimetype=field.get_mimetype())
            yield document

    @make_item('participants', wrapwith=list)
    def gen_participants(self):
        participant_fields = self.get_config_value('PUPA_PARTICIPANTS')
        for entity_type, keys in participant_fields.items():
            for key in keys:
                cell = self.field_data[key]
                participant = dict(name=cell.text, type=entity_type)
                yield participant

    @make_item('sources', wrapwith=list)
    def gen_sources(self):
        grouped = collections.defaultdict(set)
        for note, url in self.chainmap['sources'].items():
            grouped[url].add(note)
        for url, notes in grouped.items():
            yield dict(url=url, note=', '.join(notes))


class SearchView(SearchView):
    PUPATYPE = 'events'
    VIEWTYPE = 'search'
    sources_note = 'Events search'


class SearchTableRow(TableRow, EventFields):
    KEY_PREFIX = 'EVT_TABLE'

    def get_detail_url(self):
        return self.get_field_url('details')

    def get_ical_data(self):
        if hasattr(self, '_cal_data'):
            return self._ical_data
        # Don't fetch the ical data if we're testing.
        if self.cfg.USING_TEST_CONFIG:
            raise self.SkipItem()
        ical_url = self.get_ical_url()
        self.debug('%r is fetching ical data: %r', self, ical_url)
        resp = self.cfg.client.session.get(ical_url)
        with DictSetDefault(self.chainmap, 'sources', {}) as sources:
            sources['Event icalendar data (end date)'] = ical_url
        self._ical_data = resp.text
        return self._ical_data

    def get_ical_url(self):
        return self.get_field_url('ical')

    @make_item('name')
    def get_name(self):
        '''The pupa name, or Legistar "topic" of the meeting.
        '''
        return self.get_field_text('topic')

    @make_item('end')
    def get_end(self):
        '''Get the event end date from the ical record.
        '''
        end_time = re.search(r'DTEND:([\dT]+)', self.ical_data).group(1)
        dt = datetime.strptime(end_time, r'%Y%m%dT%H%M%S')
        return dt

    @make_item('when')
    def get_when(self):
        '''Get the event start date from the ical record.
        '''
        end_time = re.search(r'DTSTART:([\dT]+)', self.ical_data).group(1)
        dt = datetime.strptime(end_time, r'%Y%m%dT%H%M%S')
        return dt

{'fraction_non_alphanumeric': 0.10818933132982719, 'fraction_numerical': 0.0006010518407212623, 'mean_word_length': 2.54419595314164, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3173024', 'n_tokens_mistral': 2076, 'n_tokens_neox': 2000, 'n_words': 477}