# Path: memsql_loader/execution/errors.py
# class WorkerException(Exception):
#     def __init__(self, *args, **kwargs):
#         super(Exception, self).__init__(*args, **kwargs)
#         self.time = time.time()
# 
# class ConnectionException(WorkerException):
#     pass
# 
# class RequeueTask(Exception):
#     pass
# 
# Path: memsql_loader/util/log.py
# def setup(log_path=None, stdout_enabled=True):
# def update_verbosity(debug=False, extra_verbose=False):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
# def configure(parser):
# def get_logger(name, level=None):
#     def format(self, record):
#     def __init__(self, name, level=logging.DEBUG):
# class _SetDebug(argparse.Action):
# class _SetStdout(argparse.Action):
# class _SetLogPath(argparse.Action):
# class Formatter(logging.Formatter):
# class Logger(logging.Logger):
# 
# Path: memsql_loader/util/webhdfs.py
# def get_webhdfs_url(hdfs_host, webhdfs_port, hdfs_user, op, path):
# 
# Path: memsql_loader/util/attr_dict.py
# class AttrDict(dict):
#     def __getattr__(self, key):
#         try:
#             return self.__getitem__(key)
#         except KeyError:
#             # This lets you use dict-type attributes that aren't keys
#             return getattr(super(AttrDict, self), key)
# 
#     def __setattr__(self, key, value):
#         return self.__setitem__(key, value)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     def __repr__(self):
#         return '%s(%s)' % (self.__class__.__name__, dict.__repr__(self))
# 
#     @staticmethod
#     def from_dict(source):
#         def _transform(d):
#             """ Turns a nested dict into nested AttrDict's """
#             for k, v in d.iteritems():
#                 if isinstance(v, dict):
#                     d[k] = _transform(v)
#             return AttrDict(d)
# 
#         return _transform(source)


import pycurl
import subprocess
import select
import sys
import threading
import time
import os
import zlib
import pywebhdfs.errors
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from memsql_loader.execution.errors import WorkerException, ConnectionException, RequeueTask
from memsql_loader.util import log, webhdfs
from wraptor.decorators import throttle
from memsql_loader.util.attr_dict import AttrDict
from memsql_loader.vendor import glob2
from pywebhdfs.webhdfs import PyWebHdfsClient
        if self.key is None:
            raise WorkerException('Failed to find key associated with task ID %s' % task.task_id)

        self.metrics = DownloadMetrics(self.key.size)

    def run(self):
        try:
            try:
                # This is at the top so that any exceptions that occur will
                # emit a KILL QUERY due to fifo.open()

                # if we are piping through a script, the fifo should block
                # because the downloader is polling the script's stdin instead
                # of the fifo
                blocking = self.job.spec.options.script is not None
                with self.fifo.open(blocking=blocking) as target_file:
                    # allocate an URL for the target file
                    if self.task.data['scheme'] == 's3':
                        if self.is_anonymous:
                            key_url = 'http://%(bucket)s.s3.amazonaws.com/%(path)s' % {
                                'bucket': self.key.bucket.name,
                                'path': self.key.name.encode('utf-8')
                            }
                        else:
                            key_url = self.key.generate_url(expires_in=3600)
                    elif self.task.data['scheme'] == 'hdfs':
                        host = self.job.spec.source.hdfs_host
                        port = self.job.spec.source.webhdfs_port
                        hdfs_user = self.job.spec.source.hdfs_user
                        key_name = self.key.name

# Path: memsql_loader/execution/errors.py
# class WorkerException(Exception):
#     def __init__(self, *args, **kwargs):
#         super(Exception, self).__init__(*args, **kwargs)
#         self.time = time.time()
# 
# class ConnectionException(WorkerException):
#     pass
# 
# class RequeueTask(Exception):
#     pass
# 
# Path: memsql_loader/util/log.py
# def setup(log_path=None, stdout_enabled=True):
# def update_verbosity(debug=False, extra_verbose=False):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
# def configure(parser):
# def get_logger(name, level=None):
#     def format(self, record):
#     def __init__(self, name, level=logging.DEBUG):
# class _SetDebug(argparse.Action):
# class _SetStdout(argparse.Action):
# class _SetLogPath(argparse.Action):
# class Formatter(logging.Formatter):
# class Logger(logging.Logger):
# 
# Path: memsql_loader/util/webhdfs.py
# def get_webhdfs_url(hdfs_host, webhdfs_port, hdfs_user, op, path):
# 
# Path: memsql_loader/util/attr_dict.py
# class AttrDict(dict):
#     def __getattr__(self, key):
#         try:
#             return self.__getitem__(key)
#         except KeyError:
#             # This lets you use dict-type attributes that aren't keys
#             return getattr(super(AttrDict, self), key)
# 
#     def __setattr__(self, key, value):
#         return self.__setitem__(key, value)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     def __repr__(self):
#         return '%s(%s)' % (self.__class__.__name__, dict.__repr__(self))
# 
#     @staticmethod
#     def from_dict(source):
#         def _transform(d):
#             """ Turns a nested dict into nested AttrDict's """
#             for k, v in d.iteritems():
#                 if isinstance(v, dict):
#                     d[k] = _transform(v)
#             return AttrDict(d)
# 
#         return _transform(source)


# Path: memsql_loader/execution/downloader.py
import pycurl
import subprocess
import select
import sys
import threading
import time
import os
import zlib
import pywebhdfs.errors
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from memsql_loader.execution.errors import WorkerException, ConnectionException, RequeueTask
from memsql_loader.util import log, webhdfs
from wraptor.decorators import throttle
from memsql_loader.util.attr_dict import AttrDict
from memsql_loader.vendor import glob2
from pywebhdfs.webhdfs import PyWebHdfsClient

        if self.key is None:
            raise WorkerException('Failed to find key associated with task ID %s' % task.task_id)

        self.metrics = DownloadMetrics(self.key.size)

    def run(self):
        try:
            try:
                # This is at the top so that any exceptions that occur will
                # emit a KILL QUERY due to fifo.open()

                # if we are piping through a script, the fifo should block
                # because the downloader is polling the script's stdin instead
                # of the fifo
                blocking = self.job.spec.options.script is not None
                with self.fifo.open(blocking=blocking) as target_file:
                    # allocate an URL for the target file
                    if self.task.data['scheme'] == 's3':
                        if self.is_anonymous:
                            key_url = 'http://%(bucket)s.s3.amazonaws.com/%(path)s' % {
                                'bucket': self.key.bucket.name,
                                'path': self.key.name.encode('utf-8')
                            }
                        else:
                            key_url = self.key.generate_url(expires_in=3600)
                    elif self.task.data['scheme'] == 'hdfs':
                        host = self.job.spec.source.hdfs_host
                        port = self.job.spec.source.webhdfs_port
                        hdfs_user = self.job.spec.source.hdfs_user
                        key_name = self.key.name

                        key_url = webhdfs.get_webhdfs_url(