# Path: pupa/settings.py
# DATABASE_URL = os.environ.get('DATABASE_URL', 'postgis://pupa:pupa@localhost/opencivicdata')
# SECRET_KEY = 'non-secret'
# INSTALLED_APPS = ('django.contrib.contenttypes',
#                   'opencivicdata.core.apps.BaseConfig',
#                   'opencivicdata.legislative.apps.BaseConfig',
#                   'pupa')
# SCRAPELIB_RPM = 60
# SCRAPELIB_TIMEOUT = 60
# SCRAPELIB_RETRY_ATTEMPTS = 3
# SCRAPELIB_RETRY_WAIT_SECONDS = 10
# SCRAPELIB_VERIFY = True
# CACHE_DIR = os.path.join(os.getcwd(), '_cache')
# SCRAPED_DATA_DIR = os.path.join(os.getcwd(), '_data')
# ENABLE_PEOPLE_AND_ORGS = True
# ENABLE_BILLS = True
# ENABLE_VOTES = True
# ENABLE_EVENTS = True
# IMPORT_TRANSFORMERS = {
#     'bill': []
# }
# DEBUG = False
# TEMPLATE_DEBUG = False
# MIDDLEWARE_CLASSES = ()
# LOGGING = {
#     'version': 1,
#     'disable_existing_loggers': False,
#     'formatters': {
#         'standard': {
#             'format': "%(asctime)s %(levelname)s %(name)s: %(message)s",
#             'datefmt': '%H:%M:%S'
#         }
#     },
#     'handlers': {
#         'default': {'level': 'DEBUG',
#                     'class': 'pupa.ext.ansistrm.ColorizingStreamHandler',
#                     'formatter': 'standard'},
#     },
#     'loggers': {
#         '': {
#             'handlers': ['default'], 'level': 'DEBUG', 'propagate': True
#         },
#         'scrapelib': {
#             'handlers': ['default'], 'level': 'INFO', 'propagate': False
#         },
#         'requests': {
#             'handlers': ['default'], 'level': 'WARN', 'propagate': False
#         },
#         'boto': {
#             'handlers': ['default'], 'level': 'WARN', 'propagate': False
#         },
#     },
# }
# DATABASES = {'default': dj_database_url.parse(DATABASE_URL)}
# 
# Path: pupa/exceptions.py
# class DuplicateItemError(DataImportError):
#     """ Attempt was made to import items that resolve to the same database item. """
# 
#     def __init__(self, data, obj, data_sources=None):
#         super(DuplicateItemError, self).__init__(
#             'attempt to import data that would conflict with '
#             'data already in the import: {} '
#             '(already imported as {})\n'
#             'obj1 sources: {}\nobj2 sources: {}'.format(
#                 data,
#                 obj,
#                 list(obj.sources.values_list('url', flat=True)
#                      if hasattr(obj, 'sources') else []),
#                 [s['url'] for s in data_sources or []]
#             ))
# 
# Path: pupa/utils/generic.py
# def get_pseudo_id(pid):
#     if pid[0] != '~':
#         raise ValueError("pseudo id doesn't start with ~")
#     return json.loads(pid[1:])
# 
# def utcnow():
#     return datetime.datetime.now(datetime.timezone.utc)
# 
# Path: pupa/exceptions.py
# class UnresolvedIdError(DataImportError):
#     """ Attempt was made to resolve an id that has no result. """
# 
# class DataImportError(PupaError):
#     """ A generic error related to the import process. """
# 
# Path: pupa/models.py
# class Identifier(models.Model):
#     identifier = models.CharField(max_length=300)
#     jurisdiction = models.ForeignKey(Jurisdiction,
#                                      related_name='pupa_ids',
#                                      on_delete=models.CASCADE,
#                                      )
#     content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
#     object_id = models.CharField(max_length=300)
#     content_object = GenericForeignKey('content_type', 'object_id')
# 
#     def __str__(self):              # __unicode__ on Python 2
#         return self.identifier


import os
import copy
import glob
import json
import logging
from django.db.models import Q
from django.db.models.signals import post_save
from django.contrib.contenttypes.models import ContentType
from opencivicdata.legislative.models import LegislativeSession
from pupa import settings
from pupa.exceptions import DuplicateItemError
from pupa.utils import get_pseudo_id, utcnow
from pupa.exceptions import UnresolvedIdError, DataImportError
from pupa.models import Identifier
            # map duplicates (using omnihash to tell if json dicts are identical-ish)
            objhash = omnihash(data)
            if objhash not in seen_hashes:
                seen_hashes[objhash] = json_id
                yield json_id, data
            else:
                self.duplicates[json_id] = seen_hashes[objhash]

    def import_data(self, data_items):
        """ import a bunch of dicts together """
        # keep counts of all actions
        record = {
            'insert': 0, 'update': 0, 'noop': 0,
            'start': utcnow(),
            'records': {
                'insert': [],
                'update': [],
                'noop': [],
            }
        }

        for json_id, data in self._prepare_imports(data_items):
            obj_id, what = self.import_item(data)
            self.json_to_db_id[json_id] = obj_id
            record['records'][what].append(obj_id)
            record[what] += 1

        # all objects are loaded, a perfect time to do inter-object resolution and other tasks
        self.postimport()


# Path: pupa/settings.py
# DATABASE_URL = os.environ.get('DATABASE_URL', 'postgis://pupa:pupa@localhost/opencivicdata')
# SECRET_KEY = 'non-secret'
# INSTALLED_APPS = ('django.contrib.contenttypes',
#                   'opencivicdata.core.apps.BaseConfig',
#                   'opencivicdata.legislative.apps.BaseConfig',
#                   'pupa')
# SCRAPELIB_RPM = 60
# SCRAPELIB_TIMEOUT = 60
# SCRAPELIB_RETRY_ATTEMPTS = 3
# SCRAPELIB_RETRY_WAIT_SECONDS = 10
# SCRAPELIB_VERIFY = True
# CACHE_DIR = os.path.join(os.getcwd(), '_cache')
# SCRAPED_DATA_DIR = os.path.join(os.getcwd(), '_data')
# ENABLE_PEOPLE_AND_ORGS = True
# ENABLE_BILLS = True
# ENABLE_VOTES = True
# ENABLE_EVENTS = True
# IMPORT_TRANSFORMERS = {
#     'bill': []
# }
# DEBUG = False
# TEMPLATE_DEBUG = False
# MIDDLEWARE_CLASSES = ()
# LOGGING = {
#     'version': 1,
#     'disable_existing_loggers': False,
#     'formatters': {
#         'standard': {
#             'format': "%(asctime)s %(levelname)s %(name)s: %(message)s",
#             'datefmt': '%H:%M:%S'
#         }
#     },
#     'handlers': {
#         'default': {'level': 'DEBUG',
#                     'class': 'pupa.ext.ansistrm.ColorizingStreamHandler',
#                     'formatter': 'standard'},
#     },
#     'loggers': {
#         '': {
#             'handlers': ['default'], 'level': 'DEBUG', 'propagate': True
#         },
#         'scrapelib': {
#             'handlers': ['default'], 'level': 'INFO', 'propagate': False
#         },
#         'requests': {
#             'handlers': ['default'], 'level': 'WARN', 'propagate': False
#         },
#         'boto': {
#             'handlers': ['default'], 'level': 'WARN', 'propagate': False
#         },
#     },
# }
# DATABASES = {'default': dj_database_url.parse(DATABASE_URL)}
# 
# Path: pupa/exceptions.py
# class DuplicateItemError(DataImportError):
#     """ Attempt was made to import items that resolve to the same database item. """
# 
#     def __init__(self, data, obj, data_sources=None):
#         super(DuplicateItemError, self).__init__(
#             'attempt to import data that would conflict with '
#             'data already in the import: {} '
#             '(already imported as {})\n'
#             'obj1 sources: {}\nobj2 sources: {}'.format(
#                 data,
#                 obj,
#                 list(obj.sources.values_list('url', flat=True)
#                      if hasattr(obj, 'sources') else []),
#                 [s['url'] for s in data_sources or []]
#             ))
# 
# Path: pupa/utils/generic.py
# def get_pseudo_id(pid):
#     if pid[0] != '~':
#         raise ValueError("pseudo id doesn't start with ~")
#     return json.loads(pid[1:])
# 
# def utcnow():
#     return datetime.datetime.now(datetime.timezone.utc)
# 
# Path: pupa/exceptions.py
# class UnresolvedIdError(DataImportError):
#     """ Attempt was made to resolve an id that has no result. """
# 
# class DataImportError(PupaError):
#     """ A generic error related to the import process. """
# 
# Path: pupa/models.py
# class Identifier(models.Model):
#     identifier = models.CharField(max_length=300)
#     jurisdiction = models.ForeignKey(Jurisdiction,
#                                      related_name='pupa_ids',
#                                      on_delete=models.CASCADE,
#                                      )
#     content_type = models.ForeignKey(ContentType, on_delete=models.CASCADE)
#     object_id = models.CharField(max_length=300)
#     content_object = GenericForeignKey('content_type', 'object_id')
# 
#     def __str__(self):              # __unicode__ on Python 2
#         return self.identifier


# Path: pupa/importers/base.py
import os
import copy
import glob
import json
import logging
from django.db.models import Q
from django.db.models.signals import post_save
from django.contrib.contenttypes.models import ContentType
from opencivicdata.legislative.models import LegislativeSession
from pupa import settings
from pupa.exceptions import DuplicateItemError
from pupa.utils import get_pseudo_id, utcnow
from pupa.exceptions import UnresolvedIdError, DataImportError
from pupa.models import Identifier

            # map duplicates (using omnihash to tell if json dicts are identical-ish)
            objhash = omnihash(data)
            if objhash not in seen_hashes:
                seen_hashes[objhash] = json_id
                yield json_id, data
            else:
                self.duplicates[json_id] = seen_hashes[objhash]

    def import_data(self, data_items):
        """ import a bunch of dicts together """
        # keep counts of all actions
        record = {
            'insert': 0, 'update': 0, 'noop': 0,
            'start': utcnow(),
            'records': {
                'insert': [],
                'update': [],
                'noop': [],
            }
        }

        for json_id, data in self._prepare_imports(data_items):
            obj_id, what = self.import_item(data)
            self.json_to_db_id[json_id] = obj_id
            record['records'][what].append(obj_id)
            record[what] += 1

        # all objects are loaded, a perfect time to do inter-object resolution and other tasks
        self.postimport()


        record['end'] = utcnow()