<commit_before>from pupa.core import db
from .base import BaseImporter


class EventImporter(BaseImporter):
    _type = 'event'

    def get_db_spec(self, event):
        spec = {
            "description": event['description'],
            "when": event['when'],
            'jurisdiction_id': event['jurisdiction_id'],
        }
        return spec


    def prepare_object_from_json(self, obj):

        def person(obj, what):
            spec = {}
            spec['session'] = obj['session']
            if 'chamber' in what:
                spec['chamber'] = what['chamber']
            spec['name'] = what['name']
            # needs to get the right session (current)
            return spec

        def bill(obj, what):
            spec = {}
            spec['session'] = obj['session']
            if 'chamber' in what:
                spec['chamber'] = what['chamber']
            spec['bill_id'] = what['name']
            # needs to get the right session (current)
            return spec

        def org(obj, what):
            spec = {}
            spec['session'] = obj['session']
            if 'chamber' in what:
                spec['chamber'] = what['chamber']
            spec['name'] = what['name']
            # needs to get the right session (current)
            return spec

        spec_generators = {
            "person": person,
            "bill": bill,
            "organization": org,
        }

        # XXX participants

        for item in obj['agenda']:
            for entity in item['related_entities']:
                handler = spec_generators[entity['type']]
                spec = handler(obj, entity)
                spec['jurisdiction_id'] = obj['jurisdiction_id']
                rel_obj = db.events.find_one(spec)
                if rel_obj:
                    entity['id'] = rel_obj['_id']
                else:
                    self.logger.warning('Unknown related entity: {name} '
                                        '({type})'.format(**entity))
        return obj<commit_msg>a fix for 'when' on events<commit_after>import datetime
from pupa.core import db
from .base import BaseImporter


class EventImporter(BaseImporter):
    _type = 'event'

    def get_db_spec(self, event):
        spec = {
            "description": event['description'],
            "when": event['when'],
            'jurisdiction_id': event['jurisdiction_id'],
        }
        return spec


    def prepare_object_from_json(self, obj):

        def person(obj, what):
            spec = {}
            spec['session'] = obj['session']
            if 'chamber' in what:
                spec['chamber'] = what['chamber']
            spec['name'] = what['name']
            # needs to get the right session (current)
            return spec

        def bill(obj, what):
            spec = {}
            spec['session'] = obj['session']
            if 'chamber' in what:
                spec['chamber'] = what['chamber']
            spec['bill_id'] = what['name']
            # needs to get the right session (current)
            return spec

        def org(obj, what):
            spec = {}
            spec['session'] = obj['session']
            if 'chamber' in what:
                spec['chamber'] = what['chamber']
            spec['name'] = what['name']
            # needs to get the right session (current)
            return spec

        spec_generators = {
            "person": person,
            "bill": bill,
            "organization": org,
        }

        # XXX participants

        for item in obj['agenda']:
            for entity in item['related_entities']:
                handler = spec_generators[entity['type']]
                spec = handler(obj, entity)
                spec['jurisdiction_id'] = obj['jurisdiction_id']
                rel_obj = db.events.find_one(spec)
                if rel_obj:
                    entity['id'] = rel_obj['_id']
                else:
                    self.logger.warning('Unknown related entity: {name} '
                                        '({type})'.format(**entity))

        # update time
        obj['when'] = datetime.datetime.fromtimestamp(obj['when'])
        # TODO: handle timezones better

        return obj
{'fraction_non_alphanumeric': 0.1213982050070855, 'fraction_numerical': 0.0, 'mean_word_length': 1.5496688741721854, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3592378', 'n_tokens_mistral': 1140, 'n_tokens_neox': 1073, 'n_words': 282}