# Path: muninn/_compat.py
# PY3 = sys.version_info[0] == 3
#     def is_python2_unicode(x):
#     def dictkeys(d):
#     def dictvalues(d):
#     def path_utf8(path):
#     def encode(s):
#     def decode(s):
#     def is_python2_unicode(x):
#     def path_utf8(path):
#     def decode(s):
#     def encode(s):
# 
# Path: muninn/core.py
# class Core(Mapping):
#     uuid = UUID()
#     active = Boolean(index=True)
#     hash = Text(optional=True, index=True)
#     size = Long(optional=True, index=True)
#     metadata_date = Timestamp(index=True)
#     archive_date = Timestamp(optional=True, index=True)
#     archive_path = ArchivePath(optional=True)
#     product_type = Text(index=True)
#     product_name = Text(index=True)
#     physical_name = Basename(index=True)
#     validity_start = Timestamp(optional=True, index=True)
#     validity_stop = Timestamp(optional=True, index=True)
#     creation_date = Timestamp(optional=True, index=True)
#     footprint = Geometry(optional=True, index=True)
#     remote_url = Remote(optional=True)
# 
# Path: muninn/extension.py
# class CascadeRule(Enum):
#     _items = ("IGNORE", "CASCADE_PURGE_AS_STRIP", "CASCADE_PURGE", "STRIP", "CASCADE", "PURGE")
# 
# Path: muninn/struct.py
# class Struct(object):
#     def __init__(self, data=None):
#         super(Struct, self).__init__()
#         if data is not None:
#             for key in data:
#                 if type(data[key]) == dict:
#                     self[key] = Struct(data[key])
#                 else:
#                     self[key] = data[key]
# 
#     def __getitem__(self, key):
#         try:
#             return getattr(self, key)
#         except AttributeError:
#             raise KeyError(key)
# 
#     def __setitem__(self, key, value):
#         setattr(self, key, value)
# 
#     def __delitem__(self, key):
#         try:
#             return delattr(self, key)
#         except AttributeError:
#             raise KeyError(key)
# 
#     def __contains__(self, key):
#         return hasattr(self, key)
# 
#     def __len__(self):
#         return len(vars(self))
# 
#     def __iter__(self):
#         return iter(vars(self))
# 
#     def __repr__(self):
#         return "Struct(%r)" % vars(self)
# 
#     def update(self, other):
#         '''update a struct, using the same semantics as dict.update()'''
#         for key in other:
#             other_item = other[key]
#             if isinstance(other_item, Struct):
#                 if key not in self:
#                     self[key] = Struct()
#                 else:
#                     if not isinstance(self[key], Struct):
#                         raise Error('Incompatible structs: %s vs %s' % (self, other))
#                 self[key].update(other_item)
#             else:
#                 self[key] = other_item
# 
# Path: muninn/remote.py
# class DownloadError(Error):
# class RemoteBackend(object):
# class FileBackend(RemoteBackend):
# class HTTPBackend(RemoteBackend):
# class FTPBackend(RemoteBackend):
# def get_credentials(archive, url):
# def download_http_oath2(url, target_dir, credentials, timeout=60):
# def download_http(url, target_dir, credentials=None, timeout=60):
# def download_ftp(url, target_dir, credentials=None, timeout=60):
#     def __init__(self, prefix):
#     def indentify(self, url):
#     def auto_extract(self, file_path, product):
#     def pull(self, archive, product, target_dir):
#     def pull(self, archive, product, target_dir):
#     def pull(self, archive, product, target_dir):
# def retrieve_function(archive, product, verify_hash_download):
#     def retrieve_files(target_dir):
# REMOTE_BACKENDS = {
#     'http': HTTPBackend(prefix='http://'),
#     'https': HTTPBackend(prefix='https://'),
#     'file': FileBackend(prefix='file://'),
#     'ftp': FTPBackend(prefix='ftp://'),
# }


from muninn._compat import string_types as basestring
from muninn.core import Core
from muninn.exceptions import *
from muninn.extension import CascadeRule
from muninn.schema import *
from muninn.struct import Struct
from muninn import remote
import collections
import copy
import datetime
import errno
import functools
import inspect
import hashlib
import os
import re
import sys
import uuid
import warnings
import muninn.config as config
import muninn.util as util
            if 'remote_url' not in product.core:
                raise Error("product '%s' (%s) does not have a remote_url" %
                            (product.core.product_name, product.core.uuid))

            plugin = self.product_type_plugin(product.core.product_type)
            product.core.archive_path = plugin.archive_path(product)
            use_enclosing_directory = plugin.use_enclosing_directory

            # set archive_path and deactivate while we pull it in
            metadata = {'active': False, 'archive_path': product.core.archive_path}
            self.update_properties(Struct({'core': metadata}), product.core.uuid)

            def _pull(paths):
                # update archive_date, size
                product_path = self._product_path(product)
                size = self._storage.size(product_path)
                metadata = {'archive_date': self._database.server_time_utc(), 'size': size}
                self.update_properties(Struct({'core': metadata}), product.core.uuid)

                # verify product hash.
                if verify_hash and 'hash' in product.core:
                    if self.verify_hash("uuid == @uuid", {"uuid": product.core.uuid}):
                        raise Error("pulled product '%s' (%s) has incorrect hash" %
                                    (product.core.product_name, product.core.uuid))

                # Run the post pull hook (if defined by the product type plug-in or hook extensions).
                self._run_hooks('post_pull_hook', plugin, product, paths=paths)

            # pull product
            try:

# Path: muninn/_compat.py
# PY3 = sys.version_info[0] == 3
#     def is_python2_unicode(x):
#     def dictkeys(d):
#     def dictvalues(d):
#     def path_utf8(path):
#     def encode(s):
#     def decode(s):
#     def is_python2_unicode(x):
#     def path_utf8(path):
#     def decode(s):
#     def encode(s):
# 
# Path: muninn/core.py
# class Core(Mapping):
#     uuid = UUID()
#     active = Boolean(index=True)
#     hash = Text(optional=True, index=True)
#     size = Long(optional=True, index=True)
#     metadata_date = Timestamp(index=True)
#     archive_date = Timestamp(optional=True, index=True)
#     archive_path = ArchivePath(optional=True)
#     product_type = Text(index=True)
#     product_name = Text(index=True)
#     physical_name = Basename(index=True)
#     validity_start = Timestamp(optional=True, index=True)
#     validity_stop = Timestamp(optional=True, index=True)
#     creation_date = Timestamp(optional=True, index=True)
#     footprint = Geometry(optional=True, index=True)
#     remote_url = Remote(optional=True)
# 
# Path: muninn/extension.py
# class CascadeRule(Enum):
#     _items = ("IGNORE", "CASCADE_PURGE_AS_STRIP", "CASCADE_PURGE", "STRIP", "CASCADE", "PURGE")
# 
# Path: muninn/struct.py
# class Struct(object):
#     def __init__(self, data=None):
#         super(Struct, self).__init__()
#         if data is not None:
#             for key in data:
#                 if type(data[key]) == dict:
#                     self[key] = Struct(data[key])
#                 else:
#                     self[key] = data[key]
# 
#     def __getitem__(self, key):
#         try:
#             return getattr(self, key)
#         except AttributeError:
#             raise KeyError(key)
# 
#     def __setitem__(self, key, value):
#         setattr(self, key, value)
# 
#     def __delitem__(self, key):
#         try:
#             return delattr(self, key)
#         except AttributeError:
#             raise KeyError(key)
# 
#     def __contains__(self, key):
#         return hasattr(self, key)
# 
#     def __len__(self):
#         return len(vars(self))
# 
#     def __iter__(self):
#         return iter(vars(self))
# 
#     def __repr__(self):
#         return "Struct(%r)" % vars(self)
# 
#     def update(self, other):
#         '''update a struct, using the same semantics as dict.update()'''
#         for key in other:
#             other_item = other[key]
#             if isinstance(other_item, Struct):
#                 if key not in self:
#                     self[key] = Struct()
#                 else:
#                     if not isinstance(self[key], Struct):
#                         raise Error('Incompatible structs: %s vs %s' % (self, other))
#                 self[key].update(other_item)
#             else:
#                 self[key] = other_item
# 
# Path: muninn/remote.py
# class DownloadError(Error):
# class RemoteBackend(object):
# class FileBackend(RemoteBackend):
# class HTTPBackend(RemoteBackend):
# class FTPBackend(RemoteBackend):
# def get_credentials(archive, url):
# def download_http_oath2(url, target_dir, credentials, timeout=60):
# def download_http(url, target_dir, credentials=None, timeout=60):
# def download_ftp(url, target_dir, credentials=None, timeout=60):
#     def __init__(self, prefix):
#     def indentify(self, url):
#     def auto_extract(self, file_path, product):
#     def pull(self, archive, product, target_dir):
#     def pull(self, archive, product, target_dir):
#     def pull(self, archive, product, target_dir):
# def retrieve_function(archive, product, verify_hash_download):
#     def retrieve_files(target_dir):
# REMOTE_BACKENDS = {
#     'http': HTTPBackend(prefix='http://'),
#     'https': HTTPBackend(prefix='https://'),
#     'file': FileBackend(prefix='file://'),
#     'ftp': FTPBackend(prefix='ftp://'),
# }


# Path: muninn/archive.py
from muninn._compat import string_types as basestring
from muninn.core import Core
from muninn.exceptions import *
from muninn.extension import CascadeRule
from muninn.schema import *
from muninn.struct import Struct
from muninn import remote
import collections
import copy
import datetime
import errno
import functools
import inspect
import hashlib
import os
import re
import sys
import uuid
import warnings
import muninn.config as config
import muninn.util as util

            if 'remote_url' not in product.core:
                raise Error("product '%s' (%s) does not have a remote_url" %
                            (product.core.product_name, product.core.uuid))

            plugin = self.product_type_plugin(product.core.product_type)
            product.core.archive_path = plugin.archive_path(product)
            use_enclosing_directory = plugin.use_enclosing_directory

            # set archive_path and deactivate while we pull it in
            metadata = {'active': False, 'archive_path': product.core.archive_path}
            self.update_properties(Struct({'core': metadata}), product.core.uuid)

            def _pull(paths):
                # update archive_date, size
                product_path = self._product_path(product)
                size = self._storage.size(product_path)
                metadata = {'archive_date': self._database.server_time_utc(), 'size': size}
                self.update_properties(Struct({'core': metadata}), product.core.uuid)

                # verify product hash.
                if verify_hash and 'hash' in product.core:
                    if self.verify_hash("uuid == @uuid", {"uuid": product.core.uuid}):
                        raise Error("pulled product '%s' (%s) has incorrect hash" %
                                    (product.core.product_name, product.core.uuid))

                # Run the post pull hook (if defined by the product type plug-in or hook extensions).
                self._run_hooks('post_pull_hook', plugin, product, paths=paths)

            # pull product
            try:

                retrieve_files = remote.retrieve_function(self, product, verify_hash_download)