# Path: python/boto/glacier/writer.py
# def chunk_hashes(bytestring, chunk_size=_ONE_MEGABYTE):
#     chunk_count = int(math.ceil(len(bytestring) / float(chunk_size)))
#     hashes = []
#     for i in xrange(chunk_count):
#         start = i * chunk_size
#         end = (i + 1) * chunk_size
#         hashes.append(hashlib.sha256(bytestring[start:end]).digest())
#     return hashes
# 
# def tree_hash(fo):
#     """
#     Given a hash of each 1MB chunk (from chunk_hashes) this will hash
#     together adjacent hashes until it ends up with one big one. So a
#     tree of hashes.
#     """
#     hashes = []
#     hashes.extend(fo)
#     while len(hashes) > 1:
#         new_hashes = []
#         while True:
#             if len(hashes) > 1:
#                 first = hashes.pop(0)
#                 second = hashes.pop(0)
#                 new_hashes.append(hashlib.sha256(first + second).digest())
#             elif len(hashes) == 1:
#                 only = hashes.pop(0)
#                 new_hashes.append(only)
#             else:
#                 break
#         hashes.extend(new_hashes)
#     return hashes[0]
# 
# def bytes_to_hex(str):
#     return ''.join(["%02x" % ord(x) for x in str]).strip()
# 
# Path: python/boto/glacier/exceptions.py
# class UploadArchiveError(Exception):
#     pass


import os
import math
import threading
import hashlib
import time
import logging
from Queue import Queue, Empty
from .writer import chunk_hashes, tree_hash, bytes_to_hex
from .exceptions import UploadArchiveError
                                                       description)
        upload_id = response['UploadId']
        # The basic idea is to add the chunks (the offsets not the actual
        # contents) to a work queue, start up a thread pool, let the crank
        # through the items in the work queue, and then place their results
        # in a result queue which we use to complete the multipart upload.
        self._add_work_items_to_queue(total_parts, worker_queue)
        self._start_upload_threads(result_queue, upload_id,
                                   worker_queue, filename)
        try:
            self._wait_for_upload_threads(hash_chunks, result_queue, total_parts)
        except UploadArchiveError, e:
            log.debug("An error occurred while uploading an archive, aborting "
                      "multipart upload.")
            self._api.abort_multipart_upload(self._vault_name, upload_id)
            raise e
        log.debug("Completing upload.")
        response = self._api.complete_multipart_upload(
            self._vault_name, upload_id, bytes_to_hex(tree_hash(hash_chunks)),
            total_size)
        log.debug("Upload finished.")
        return response['ArchiveId']

    def _wait_for_upload_threads(self, hash_chunks, result_queue, total_parts):
        for _ in xrange(total_parts):
            result = result_queue.get()
            if isinstance(result, Exception):
                log.debug("An error was found in the result queue, terminating "
                          "threads: %s", result)
                self._shutdown_threads()

# Path: python/boto/glacier/writer.py
# def chunk_hashes(bytestring, chunk_size=_ONE_MEGABYTE):
#     chunk_count = int(math.ceil(len(bytestring) / float(chunk_size)))
#     hashes = []
#     for i in xrange(chunk_count):
#         start = i * chunk_size
#         end = (i + 1) * chunk_size
#         hashes.append(hashlib.sha256(bytestring[start:end]).digest())
#     return hashes
# 
# def tree_hash(fo):
#     """
#     Given a hash of each 1MB chunk (from chunk_hashes) this will hash
#     together adjacent hashes until it ends up with one big one. So a
#     tree of hashes.
#     """
#     hashes = []
#     hashes.extend(fo)
#     while len(hashes) > 1:
#         new_hashes = []
#         while True:
#             if len(hashes) > 1:
#                 first = hashes.pop(0)
#                 second = hashes.pop(0)
#                 new_hashes.append(hashlib.sha256(first + second).digest())
#             elif len(hashes) == 1:
#                 only = hashes.pop(0)
#                 new_hashes.append(only)
#             else:
#                 break
#         hashes.extend(new_hashes)
#     return hashes[0]
# 
# def bytes_to_hex(str):
#     return ''.join(["%02x" % ord(x) for x in str]).strip()
# 
# Path: python/boto/glacier/exceptions.py
# class UploadArchiveError(Exception):
#     pass


# Path: python/boto/glacier/concurrent.py
import os
import math
import threading
import hashlib
import time
import logging
from Queue import Queue, Empty
from .writer import chunk_hashes, tree_hash, bytes_to_hex
from .exceptions import UploadArchiveError

                                                       description)
        upload_id = response['UploadId']
        # The basic idea is to add the chunks (the offsets not the actual
        # contents) to a work queue, start up a thread pool, let the crank
        # through the items in the work queue, and then place their results
        # in a result queue which we use to complete the multipart upload.
        self._add_work_items_to_queue(total_parts, worker_queue)
        self._start_upload_threads(result_queue, upload_id,
                                   worker_queue, filename)
        try:
            self._wait_for_upload_threads(hash_chunks, result_queue, total_parts)
        except UploadArchiveError, e:
            log.debug("An error occurred while uploading an archive, aborting "
                      "multipart upload.")
            self._api.abort_multipart_upload(self._vault_name, upload_id)
            raise e
        log.debug("Completing upload.")
        response = self._api.complete_multipart_upload(
            self._vault_name, upload_id, bytes_to_hex(tree_hash(hash_chunks)),
            total_size)
        log.debug("Upload finished.")
        return response['ArchiveId']

    def _wait_for_upload_threads(self, hash_chunks, result_queue, total_parts):
        for _ in xrange(total_parts):
            result = result_queue.get()
            if isinstance(result, Exception):
                log.debug("An error was found in the result queue, terminating "
                          "threads: %s", result)
                self._shutdown_threads()

                raise UploadArchiveError("An error occurred while uploading "