# Path: memsql_loader/execution/errors.py
# class WorkerException(Exception):
#     def __init__(self, *args, **kwargs):
#         super(Exception, self).__init__(*args, **kwargs)
#         self.time = time.time()
# 
# class ConnectionException(WorkerException):
#     pass
# 
# class RequeueTask(Exception):
#     pass
# 
# Path: memsql_loader/util/log.py
# def setup(log_path=None, stdout_enabled=True):
# def update_verbosity(debug=False, extra_verbose=False):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
# def configure(parser):
# def get_logger(name, level=None):
#     def format(self, record):
#     def __init__(self, name, level=logging.DEBUG):
# class _SetDebug(argparse.Action):
# class _SetStdout(argparse.Action):
# class _SetLogPath(argparse.Action):
# class Formatter(logging.Formatter):
# class Logger(logging.Logger):
# 
# Path: memsql_loader/util/webhdfs.py
# def get_webhdfs_url(hdfs_host, webhdfs_port, hdfs_user, op, path):
# 
# Path: memsql_loader/util/attr_dict.py
# class AttrDict(dict):
#     def __getattr__(self, key):
#         try:
#             return self.__getitem__(key)
#         except KeyError:
#             # This lets you use dict-type attributes that aren't keys
#             return getattr(super(AttrDict, self), key)
# 
#     def __setattr__(self, key, value):
#         return self.__setitem__(key, value)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     def __repr__(self):
#         return '%s(%s)' % (self.__class__.__name__, dict.__repr__(self))
# 
#     @staticmethod
#     def from_dict(source):
#         def _transform(d):
#             """ Turns a nested dict into nested AttrDict's """
#             for k, v in d.iteritems():
#                 if isinstance(v, dict):
#                     d[k] = _transform(v)
#             return AttrDict(d)
# 
#         return _transform(source)


import pycurl
import subprocess
import select
import sys
import threading
import time
import os
import zlib
import pywebhdfs.errors
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from memsql_loader.execution.errors import WorkerException, ConnectionException, RequeueTask
from memsql_loader.util import log, webhdfs
from wraptor.decorators import throttle
from memsql_loader.util.attr_dict import AttrDict
from memsql_loader.vendor import glob2
from pywebhdfs.webhdfs import PyWebHdfsClient

        # update our last change time so long as our speed is above 10 bytes per
        # second or we have only been downloading for 30 seconds
        if diff > 10:
            self.ping()

        self._snapshots.append(diff)
        if len(self._snapshots) > self._avg_len:
            self._snapshots = self._snapshots[(-1 * self._avg_len):]

        self._last_snapshot = current

    def get_stats(self):
        if self._current_size == self._total_size or len(self._snapshots) == 0:
            rate = 0
        else:
            # Not super efficient, but shouldn't really matter with max 30 values
            rate = sum(self._snapshots) / len(self._snapshots)

        time_left = -1 if rate == 0 else (self._total_size - self._current_size) / rate

        return {
            'bytes_downloaded': self._current_size,
            'download_rate': rate,
            'time_left': time_left
        }

class Downloader(threading.Thread):
    def __init__(self):
        super(Downloader, self).__init__()

# Path: memsql_loader/execution/errors.py
# class WorkerException(Exception):
#     def __init__(self, *args, **kwargs):
#         super(Exception, self).__init__(*args, **kwargs)
#         self.time = time.time()
# 
# class ConnectionException(WorkerException):
#     pass
# 
# class RequeueTask(Exception):
#     pass
# 
# Path: memsql_loader/util/log.py
# def setup(log_path=None, stdout_enabled=True):
# def update_verbosity(debug=False, extra_verbose=False):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
# def configure(parser):
# def get_logger(name, level=None):
#     def format(self, record):
#     def __init__(self, name, level=logging.DEBUG):
# class _SetDebug(argparse.Action):
# class _SetStdout(argparse.Action):
# class _SetLogPath(argparse.Action):
# class Formatter(logging.Formatter):
# class Logger(logging.Logger):
# 
# Path: memsql_loader/util/webhdfs.py
# def get_webhdfs_url(hdfs_host, webhdfs_port, hdfs_user, op, path):
# 
# Path: memsql_loader/util/attr_dict.py
# class AttrDict(dict):
#     def __getattr__(self, key):
#         try:
#             return self.__getitem__(key)
#         except KeyError:
#             # This lets you use dict-type attributes that aren't keys
#             return getattr(super(AttrDict, self), key)
# 
#     def __setattr__(self, key, value):
#         return self.__setitem__(key, value)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     def __repr__(self):
#         return '%s(%s)' % (self.__class__.__name__, dict.__repr__(self))
# 
#     @staticmethod
#     def from_dict(source):
#         def _transform(d):
#             """ Turns a nested dict into nested AttrDict's """
#             for k, v in d.iteritems():
#                 if isinstance(v, dict):
#                     d[k] = _transform(v)
#             return AttrDict(d)
# 
#         return _transform(source)


# Path: memsql_loader/execution/downloader.py
import pycurl
import subprocess
import select
import sys
import threading
import time
import os
import zlib
import pywebhdfs.errors
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from memsql_loader.execution.errors import WorkerException, ConnectionException, RequeueTask
from memsql_loader.util import log, webhdfs
from wraptor.decorators import throttle
from memsql_loader.util.attr_dict import AttrDict
from memsql_loader.vendor import glob2
from pywebhdfs.webhdfs import PyWebHdfsClient


        # update our last change time so long as our speed is above 10 bytes per
        # second or we have only been downloading for 30 seconds
        if diff > 10:
            self.ping()

        self._snapshots.append(diff)
        if len(self._snapshots) > self._avg_len:
            self._snapshots = self._snapshots[(-1 * self._avg_len):]

        self._last_snapshot = current

    def get_stats(self):
        if self._current_size == self._total_size or len(self._snapshots) == 0:
            rate = 0
        else:
            # Not super efficient, but shouldn't really matter with max 30 values
            rate = sum(self._snapshots) / len(self._snapshots)

        time_left = -1 if rate == 0 else (self._total_size - self._current_size) / rate

        return {
            'bytes_downloaded': self._current_size,
            'download_rate': rate,
            'time_left': time_left
        }

class Downloader(threading.Thread):
    def __init__(self):
        super(Downloader, self).__init__()

        self.logger = log.get_logger('downloader')