# Path: memsql_loader/execution/errors.py
# class WorkerException(Exception):
#     def __init__(self, *args, **kwargs):
#         super(Exception, self).__init__(*args, **kwargs)
#         self.time = time.time()
# 
# class ConnectionException(WorkerException):
#     pass
# 
# class RequeueTask(Exception):
#     pass
# 
# Path: memsql_loader/util/log.py
# def setup(log_path=None, stdout_enabled=True):
# def update_verbosity(debug=False, extra_verbose=False):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
# def configure(parser):
# def get_logger(name, level=None):
#     def format(self, record):
#     def __init__(self, name, level=logging.DEBUG):
# class _SetDebug(argparse.Action):
# class _SetStdout(argparse.Action):
# class _SetLogPath(argparse.Action):
# class Formatter(logging.Formatter):
# class Logger(logging.Logger):
# 
# Path: memsql_loader/util/webhdfs.py
# def get_webhdfs_url(hdfs_host, webhdfs_port, hdfs_user, op, path):
# 
# Path: memsql_loader/util/attr_dict.py
# class AttrDict(dict):
#     def __getattr__(self, key):
#         try:
#             return self.__getitem__(key)
#         except KeyError:
#             # This lets you use dict-type attributes that aren't keys
#             return getattr(super(AttrDict, self), key)
# 
#     def __setattr__(self, key, value):
#         return self.__setitem__(key, value)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     def __repr__(self):
#         return '%s(%s)' % (self.__class__.__name__, dict.__repr__(self))
# 
#     @staticmethod
#     def from_dict(source):
#         def _transform(d):
#             """ Turns a nested dict into nested AttrDict's """
#             for k, v in d.iteritems():
#                 if isinstance(v, dict):
#                     d[k] = _transform(v)
#             return AttrDict(d)
# 
#         return _transform(source)


import pycurl
import subprocess
import select
import sys
import threading
import time
import os
import zlib
import pywebhdfs.errors
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from memsql_loader.execution.errors import WorkerException, ConnectionException, RequeueTask
from memsql_loader.util import log, webhdfs
from wraptor.decorators import throttle
from memsql_loader.util.attr_dict import AttrDict
from memsql_loader.vendor import glob2
from pywebhdfs.webhdfs import PyWebHdfsClient
                                self.logger.error('Script `%s` failed to exit...killing' % self.job.spec.options.script)
                                self.script_proc.kill()
                                raise WorkerException('Script `%s` failed to exit after %d seconds' % (self.job.spec.options.script, SCRIPT_EXIT_TIMEOUT))

                            if self.script_proc.returncode != 0:
                                self.logger.error('Script `%s` exited with return code %d' % (self.job.spec.options.script, self.script_proc.returncode))
                                raise WorkerException('Script `%s` exited with return code %d' % (self.job.spec.options.script, self.script_proc.returncode))
                    finally:
                        with self.task.protect():
                            self.task.stop_step('download')

                            if self.script_proc is not None and self.script_proc.returncode is None:
                                try:
                                    self.script_proc.kill()
                                except OSError as e:
                                    self.logger.warn("Failed to kill script `%s`: %s" % (self.job.spec.options.script, str(e)))
            except pycurl.error as e:
                errno = e.args[0]
                if errno in (pycurl.E_WRITE_ERROR, pycurl.E_ABORTED_BY_CALLBACK):
                    if self.pycurl_callback_exception is not None:
                        raise self.pycurl_callback_exception
                    elif self._should_exit:
                        self.logger.warn('Download failed...requeueing')
                        # Caught by the outer `except Exception as e`
                        raise RequeueTask()

                # Caught by the outer `except pycurl.error as e`
                raise
        except pycurl.error as e:
            errno = e.args[0]

# Path: memsql_loader/execution/errors.py
# class WorkerException(Exception):
#     def __init__(self, *args, **kwargs):
#         super(Exception, self).__init__(*args, **kwargs)
#         self.time = time.time()
# 
# class ConnectionException(WorkerException):
#     pass
# 
# class RequeueTask(Exception):
#     pass
# 
# Path: memsql_loader/util/log.py
# def setup(log_path=None, stdout_enabled=True):
# def update_verbosity(debug=False, extra_verbose=False):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
# def configure(parser):
# def get_logger(name, level=None):
#     def format(self, record):
#     def __init__(self, name, level=logging.DEBUG):
# class _SetDebug(argparse.Action):
# class _SetStdout(argparse.Action):
# class _SetLogPath(argparse.Action):
# class Formatter(logging.Formatter):
# class Logger(logging.Logger):
# 
# Path: memsql_loader/util/webhdfs.py
# def get_webhdfs_url(hdfs_host, webhdfs_port, hdfs_user, op, path):
# 
# Path: memsql_loader/util/attr_dict.py
# class AttrDict(dict):
#     def __getattr__(self, key):
#         try:
#             return self.__getitem__(key)
#         except KeyError:
#             # This lets you use dict-type attributes that aren't keys
#             return getattr(super(AttrDict, self), key)
# 
#     def __setattr__(self, key, value):
#         return self.__setitem__(key, value)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     def __repr__(self):
#         return '%s(%s)' % (self.__class__.__name__, dict.__repr__(self))
# 
#     @staticmethod
#     def from_dict(source):
#         def _transform(d):
#             """ Turns a nested dict into nested AttrDict's """
#             for k, v in d.iteritems():
#                 if isinstance(v, dict):
#                     d[k] = _transform(v)
#             return AttrDict(d)
# 
#         return _transform(source)


# Path: memsql_loader/execution/downloader.py
import pycurl
import subprocess
import select
import sys
import threading
import time
import os
import zlib
import pywebhdfs.errors
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from memsql_loader.execution.errors import WorkerException, ConnectionException, RequeueTask
from memsql_loader.util import log, webhdfs
from wraptor.decorators import throttle
from memsql_loader.util.attr_dict import AttrDict
from memsql_loader.vendor import glob2
from pywebhdfs.webhdfs import PyWebHdfsClient

                                self.logger.error('Script `%s` failed to exit...killing' % self.job.spec.options.script)
                                self.script_proc.kill()
                                raise WorkerException('Script `%s` failed to exit after %d seconds' % (self.job.spec.options.script, SCRIPT_EXIT_TIMEOUT))

                            if self.script_proc.returncode != 0:
                                self.logger.error('Script `%s` exited with return code %d' % (self.job.spec.options.script, self.script_proc.returncode))
                                raise WorkerException('Script `%s` exited with return code %d' % (self.job.spec.options.script, self.script_proc.returncode))
                    finally:
                        with self.task.protect():
                            self.task.stop_step('download')

                            if self.script_proc is not None and self.script_proc.returncode is None:
                                try:
                                    self.script_proc.kill()
                                except OSError as e:
                                    self.logger.warn("Failed to kill script `%s`: %s" % (self.job.spec.options.script, str(e)))
            except pycurl.error as e:
                errno = e.args[0]
                if errno in (pycurl.E_WRITE_ERROR, pycurl.E_ABORTED_BY_CALLBACK):
                    if self.pycurl_callback_exception is not None:
                        raise self.pycurl_callback_exception
                    elif self._should_exit:
                        self.logger.warn('Download failed...requeueing')
                        # Caught by the outer `except Exception as e`
                        raise RequeueTask()

                # Caught by the outer `except pycurl.error as e`
                raise
        except pycurl.error as e:
            errno = e.args[0]

            self._set_error(ConnectionException('libcurl error #%d. Lookup error here: http://curl.haxx.se/libcurl/c/libcurl-errors.html' % errno))