# Path: cloudvolume/connectionpools.py
# class S3ConnectionPool(ConnectionPool):
#   def __init__(self, service, bucket):
#     self.service = service
#     self.bucket = bucket
#     self.credentials = aws_credentials(bucket, service)
#     super(S3ConnectionPool, self).__init__()
# 
#   @retry
#   def _create_connection(self):
#     if self.service in ('aws', 's3'):
#       return boto3.client(
#         's3',
#         aws_access_key_id=self.credentials['AWS_ACCESS_KEY_ID'],
#         aws_secret_access_key=self.credentials['AWS_SECRET_ACCESS_KEY'],
#         region_name='us-east-1',
#       )
#     elif self.service == 'matrix':
#       return boto3.client(
#         's3',
#         aws_access_key_id=self.credentials['AWS_ACCESS_KEY_ID'],
#         aws_secret_access_key=self.credentials['AWS_SECRET_ACCESS_KEY'],
#         endpoint_url='https://s3-hpcrc.rc.princeton.edu',
#       )
#     else:
#       raise UnsupportedProtocolError("{} unknown. Choose from 's3' or 'matrix'.", self.service)
#       
#   def close(self, conn):
#     try:
#       return conn.close()
#     except AttributeError:
#       pass # AttributeError: 'S3' object has no attribute 'close' on shutdown
# 
# class GCloudBucketPool(ConnectionPool):
#   def __init__(self, bucket):
#     self.bucket = bucket
#     self.project, self.credentials = google_credentials(bucket)
#     super(GCloudBucketPool, self).__init__()
# 
#   @retry
#   def _create_connection(self):
#     client = Client(
#       credentials=self.credentials,
#       project=self.project,
#     )
# 
#     return client.bucket(self.bucket)
# 
# Path: cloudvolume/lib.py
# def mkdir(path):
#   path = toabs(path)
# 
#   try:
#     if path != '' and not os.path.exists(path):
#       os.makedirs(path)
#   except OSError as e:
#     if e.errno == 17: # File Exists
#       time.sleep(0.1)
#       return mkdir(path)
#     else:
#       raise
# 
#   return path
# 
# Path: cloudvolume/exceptions.py
# class UnsupportedCompressionType(ValueError):
#   """
#   Raised when attempting to use a compression type which is unsupported
#   by the storage interface.
#   """


import six
import json
import os.path
import posixpath
import re
import boto3
import botocore
import google.cloud.exceptions
import requests
import tenacity
  import gc
from collections import defaultdict
from glob import glob
from google.cloud.storage import Batch, Client
from cloudvolume.connectionpools import S3ConnectionPool, GCloudBucketPool
from cloudvolume.lib import mkdir
from cloudvolume.exceptions import UnsupportedCompressionType



COMPRESSION_EXTENSIONS = ('.gz', '.br')

# This is just to support pooling by bucket
class keydefaultdict(defaultdict):
  def __missing__(self, key):
    if self.default_factory is None:
      raise KeyError( key )
    else:
      ret = self[key] = self.default_factory(key)
      return ret

S3_POOL = None
GC_POOL = None
def reset_connection_pools():
  global S3_POOL
  global GC_POOL

# Path: cloudvolume/connectionpools.py
# class S3ConnectionPool(ConnectionPool):
#   def __init__(self, service, bucket):
#     self.service = service
#     self.bucket = bucket
#     self.credentials = aws_credentials(bucket, service)
#     super(S3ConnectionPool, self).__init__()
# 
#   @retry
#   def _create_connection(self):
#     if self.service in ('aws', 's3'):
#       return boto3.client(
#         's3',
#         aws_access_key_id=self.credentials['AWS_ACCESS_KEY_ID'],
#         aws_secret_access_key=self.credentials['AWS_SECRET_ACCESS_KEY'],
#         region_name='us-east-1',
#       )
#     elif self.service == 'matrix':
#       return boto3.client(
#         's3',
#         aws_access_key_id=self.credentials['AWS_ACCESS_KEY_ID'],
#         aws_secret_access_key=self.credentials['AWS_SECRET_ACCESS_KEY'],
#         endpoint_url='https://s3-hpcrc.rc.princeton.edu',
#       )
#     else:
#       raise UnsupportedProtocolError("{} unknown. Choose from 's3' or 'matrix'.", self.service)
#       
#   def close(self, conn):
#     try:
#       return conn.close()
#     except AttributeError:
#       pass # AttributeError: 'S3' object has no attribute 'close' on shutdown
# 
# class GCloudBucketPool(ConnectionPool):
#   def __init__(self, bucket):
#     self.bucket = bucket
#     self.project, self.credentials = google_credentials(bucket)
#     super(GCloudBucketPool, self).__init__()
# 
#   @retry
#   def _create_connection(self):
#     client = Client(
#       credentials=self.credentials,
#       project=self.project,
#     )
# 
#     return client.bucket(self.bucket)
# 
# Path: cloudvolume/lib.py
# def mkdir(path):
#   path = toabs(path)
# 
#   try:
#     if path != '' and not os.path.exists(path):
#       os.makedirs(path)
#   except OSError as e:
#     if e.errno == 17: # File Exists
#       time.sleep(0.1)
#       return mkdir(path)
#     else:
#       raise
# 
#   return path
# 
# Path: cloudvolume/exceptions.py
# class UnsupportedCompressionType(ValueError):
#   """
#   Raised when attempting to use a compression type which is unsupported
#   by the storage interface.
#   """


# Path: cloudvolume/storage/storage_interfaces.py
import six
import json
import os.path
import posixpath
import re
import boto3
import botocore
import google.cloud.exceptions
import requests
import tenacity
  import gc
from collections import defaultdict
from glob import glob
from google.cloud.storage import Batch, Client
from cloudvolume.connectionpools import S3ConnectionPool, GCloudBucketPool
from cloudvolume.lib import mkdir
from cloudvolume.exceptions import UnsupportedCompressionType




COMPRESSION_EXTENSIONS = ('.gz', '.br')

# This is just to support pooling by bucket
class keydefaultdict(defaultdict):
  def __missing__(self, key):
    if self.default_factory is None:
      raise KeyError( key )
    else:
      ret = self[key] = self.default_factory(key)
      return ret

S3_POOL = None
GC_POOL = None
def reset_connection_pools():
  global S3_POOL
  global GC_POOL

  S3_POOL = keydefaultdict(lambda service: keydefaultdict(lambda bucket_name: S3ConnectionPool(service, bucket_name)))