# Path: anchore/util/resources.py
# class ResourceCache(object):
#     """
#     A Resource cache backed by a local directory.
#     Composed of a storage directory with content files (named by etag) and a metadata file
#     that preserves HTTP headers and maps the content files to the URLs from whence they came. Handles the case of
#     the same content via different URLs (common with S3, for example).
#     """
#     _logger = logging.getLogger(__name__)
#     metadata_filename = 'cache_metadata.json'
# 
#     def __init__(self, path='/tmp', progress_bars=False):
#         self.path = path
#         self.progress_bars = progress_bars
#         #self.progress_bars = False
# 
#         self.metadata_file = os.path.join(self.path, self.metadata_filename)
#         self.metadata = {}
# 
#         if not os.path.exists(self.metadata_file):
#             self._logger.debug('Initializing resource cache at %s' % self.path)
#             fs_util.createpath(path=self.path, mode=0744, exists_ok=True)
#             with open(self.metadata_file, 'w') as f:
#                 json.dump(self.metadata, f)
#         else:
#             with open(self.metadata_file) as f:
#                 self.metadata = json.load(f)
# 
#         self._validate_cache()
# 
#     def _validate_cache(self):
#         for url, entry in self.metadata.items():
#             # Remove any where file is not found. Can occur due to use Ctrl-C or unexpected failure
#             if not os.path.exists(entry['content']):
#                 self.metadata.pop(url)
# 
#     def lookup(self, url):
#         """
#         Like 'get', but only returns metadata if present. Will *not* download/load the cache
#         :param url:
#         :return:
#         """
#         return self.metadata[url]
# 
#     def check(self, url):
#         return url in self.metadata
# 
#     def is_stale(self, url):
#         self._logger.debug('Checking staleness of url %s into resource cache' % url)
#         retriever = get_resource_retriever(url)
# 
#         try:
#             if url in self.metadata:
#                 headers = retriever.get_metadata()
#                 if 'etag' in headers:
#                     return self.metadata[url]['etag'] != headers['etag']
#                 elif 'lastmodified' in headers:
#                     return self.metadata[url]['lastmodified'] != headers['lastmodified']
# 
#             else:
#                 return True
#         except Exception, e:
#             self._logger.debug('Error checking staleness of %s: %s' % (url, e.message))
#             raise
# 
#     def get(self, url):
#         """
#         Lookup the given url and return an entry if found. Else return None.
#         The returned entry is a dict with metadata about the content and a 'content' key with a file path value
# 
#         :param url:
#         :return:
#         """
#         self._load(url)
#         self._flush()
# 
#         return self.metadata[url]
# 
#     def is_empty(self):
#         return len(self.metadata) == 0
# 
#     def clear(self):
#         """
#         Remove all entries from the cache and delete all data.
#         :return:
#         """
#         for f in [x['content'] for x in self.metadata.values()]:
#             os.remove(f)
# 
#         self.metadata = {}
#         self._flush()
# 
#     @staticmethod
#     def _hash_path(url):
#         return hashlib.md5(url).hexdigest()
# 
#     def _load(self, url):
#         """
#         Load from remote, but check local file content to identify duplicate content. If local file is found with
#         same hash then it is used with metadata from remote object to avoid fetching full content.
# 
#         :param url:
#         :return:
#         """
#         self._logger.debug('Loading url %s into resource cache' % url)
#         retriever = get_resource_retriever(url)
#         content_path = os.path.join(self.path, self._hash_path(url))
# 
#         try:
#             if url in self.metadata:
#                 headers = retriever.fetch(content_path, last_etag=self.metadata[url]['etag'], progress_bar=self.progress_bars)
#                 if headers is None:
#                     # no update, return what is already loaded
#                     self._logger.info('Cached %s is up-to-date. No data download needed' % url)
#                     return self.metadata[url]
#             else:
#                 headers = retriever.fetch(content_path, progress_bar=self.progress_bars)
#                 if headers is None:
#                     raise Exception('Fetch of %s failed' % url)
# 
#             if 'etag' not in headers:
#                 # No Etag returned, so generate it
#                 headers['etag'] = fs_util.calc_file_md5(content_path)
# 
#             # Populate metadata from the headers
#             self.metadata[url] = headers.copy()
#             self.metadata[url]['content'] = content_path
#             return self.metadata[url]
#         except:
#             self._logger.error('Failed getting resource: %s')
# 
#             # forcibly flush local entry if found
#             if url in self.metadata:
#                 self.metadata.pop(url)
#             raise
#         finally:
#             if url not in self.metadata:
#                 self._logger.debug('Cleaning up on failed load of %s' % url)
#                 # Cleanup on failure
#                 if content_path is not None and os.path.exists(content_path):
#                     os.remove(content_path)
# 
#     def _flush(self):
#         """
#         Flush metadata to the backing file
#         :return:
#         """
#         with open(self.metadata_file, 'w') as f:
#             json.dump(self.metadata, f)
# 
#     #def __del__(self):
#         # Ensure the lock is released
#         #fcntl.flock(self._metadata_fd, fcntl.LOCK_UN)


import unittest
import logging
import os
from anchore.util.resources import ResourceCache


logging.basicConfig(level='DEBUG')


class TestResourceCache (unittest.TestCase):
    def test_basic(self):
        cache = ResourceCache(directory='/tmp')
        cache.clear()
        assert cache.is_empty()

        entry = cache.get('http://downloads.anchore.s3-website-us-west-2.amazonaws.com/')
        print entry
        assert entry['content'] is not None

        print cache.metadata
        cache.clear()

    def test_etagmatch(self):

# Path: anchore/util/resources.py
# class ResourceCache(object):
#     """
#     A Resource cache backed by a local directory.
#     Composed of a storage directory with content files (named by etag) and a metadata file
#     that preserves HTTP headers and maps the content files to the URLs from whence they came. Handles the case of
#     the same content via different URLs (common with S3, for example).
#     """
#     _logger = logging.getLogger(__name__)
#     metadata_filename = 'cache_metadata.json'
# 
#     def __init__(self, path='/tmp', progress_bars=False):
#         self.path = path
#         self.progress_bars = progress_bars
#         #self.progress_bars = False
# 
#         self.metadata_file = os.path.join(self.path, self.metadata_filename)
#         self.metadata = {}
# 
#         if not os.path.exists(self.metadata_file):
#             self._logger.debug('Initializing resource cache at %s' % self.path)
#             fs_util.createpath(path=self.path, mode=0744, exists_ok=True)
#             with open(self.metadata_file, 'w') as f:
#                 json.dump(self.metadata, f)
#         else:
#             with open(self.metadata_file) as f:
#                 self.metadata = json.load(f)
# 
#         self._validate_cache()
# 
#     def _validate_cache(self):
#         for url, entry in self.metadata.items():
#             # Remove any where file is not found. Can occur due to use Ctrl-C or unexpected failure
#             if not os.path.exists(entry['content']):
#                 self.metadata.pop(url)
# 
#     def lookup(self, url):
#         """
#         Like 'get', but only returns metadata if present. Will *not* download/load the cache
#         :param url:
#         :return:
#         """
#         return self.metadata[url]
# 
#     def check(self, url):
#         return url in self.metadata
# 
#     def is_stale(self, url):
#         self._logger.debug('Checking staleness of url %s into resource cache' % url)
#         retriever = get_resource_retriever(url)
# 
#         try:
#             if url in self.metadata:
#                 headers = retriever.get_metadata()
#                 if 'etag' in headers:
#                     return self.metadata[url]['etag'] != headers['etag']
#                 elif 'lastmodified' in headers:
#                     return self.metadata[url]['lastmodified'] != headers['lastmodified']
# 
#             else:
#                 return True
#         except Exception, e:
#             self._logger.debug('Error checking staleness of %s: %s' % (url, e.message))
#             raise
# 
#     def get(self, url):
#         """
#         Lookup the given url and return an entry if found. Else return None.
#         The returned entry is a dict with metadata about the content and a 'content' key with a file path value
# 
#         :param url:
#         :return:
#         """
#         self._load(url)
#         self._flush()
# 
#         return self.metadata[url]
# 
#     def is_empty(self):
#         return len(self.metadata) == 0
# 
#     def clear(self):
#         """
#         Remove all entries from the cache and delete all data.
#         :return:
#         """
#         for f in [x['content'] for x in self.metadata.values()]:
#             os.remove(f)
# 
#         self.metadata = {}
#         self._flush()
# 
#     @staticmethod
#     def _hash_path(url):
#         return hashlib.md5(url).hexdigest()
# 
#     def _load(self, url):
#         """
#         Load from remote, but check local file content to identify duplicate content. If local file is found with
#         same hash then it is used with metadata from remote object to avoid fetching full content.
# 
#         :param url:
#         :return:
#         """
#         self._logger.debug('Loading url %s into resource cache' % url)
#         retriever = get_resource_retriever(url)
#         content_path = os.path.join(self.path, self._hash_path(url))
# 
#         try:
#             if url in self.metadata:
#                 headers = retriever.fetch(content_path, last_etag=self.metadata[url]['etag'], progress_bar=self.progress_bars)
#                 if headers is None:
#                     # no update, return what is already loaded
#                     self._logger.info('Cached %s is up-to-date. No data download needed' % url)
#                     return self.metadata[url]
#             else:
#                 headers = retriever.fetch(content_path, progress_bar=self.progress_bars)
#                 if headers is None:
#                     raise Exception('Fetch of %s failed' % url)
# 
#             if 'etag' not in headers:
#                 # No Etag returned, so generate it
#                 headers['etag'] = fs_util.calc_file_md5(content_path)
# 
#             # Populate metadata from the headers
#             self.metadata[url] = headers.copy()
#             self.metadata[url]['content'] = content_path
#             return self.metadata[url]
#         except:
#             self._logger.error('Failed getting resource: %s')
# 
#             # forcibly flush local entry if found
#             if url in self.metadata:
#                 self.metadata.pop(url)
#             raise
#         finally:
#             if url not in self.metadata:
#                 self._logger.debug('Cleaning up on failed load of %s' % url)
#                 # Cleanup on failure
#                 if content_path is not None and os.path.exists(content_path):
#                     os.remove(content_path)
# 
#     def _flush(self):
#         """
#         Flush metadata to the backing file
#         :return:
#         """
#         with open(self.metadata_file, 'w') as f:
#             json.dump(self.metadata, f)
# 
#     #def __del__(self):
#         # Ensure the lock is released
#         #fcntl.flock(self._metadata_fd, fcntl.LOCK_UN)


# Path: tests/unit/test_resourcecache.py
import unittest
import logging
import os
from anchore.util.resources import ResourceCache



logging.basicConfig(level='DEBUG')


class TestResourceCache (unittest.TestCase):
    def test_basic(self):
        cache = ResourceCache(directory='/tmp')
        cache.clear()
        assert cache.is_empty()

        entry = cache.get('http://downloads.anchore.s3-website-us-west-2.amazonaws.com/')
        print entry
        assert entry['content'] is not None

        print cache.metadata
        cache.clear()

    def test_etagmatch(self):

        cache = ResourceCache(directory='/tmp')