# Path: memsql_loader/execution/errors.py
# class WorkerException(Exception):
#     def __init__(self, *args, **kwargs):
#         super(Exception, self).__init__(*args, **kwargs)
#         self.time = time.time()
# 
# class ConnectionException(WorkerException):
#     pass
# 
# class RequeueTask(Exception):
#     pass
# 
# Path: memsql_loader/util/log.py
# def setup(log_path=None, stdout_enabled=True):
# def update_verbosity(debug=False, extra_verbose=False):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
# def configure(parser):
# def get_logger(name, level=None):
#     def format(self, record):
#     def __init__(self, name, level=logging.DEBUG):
# class _SetDebug(argparse.Action):
# class _SetStdout(argparse.Action):
# class _SetLogPath(argparse.Action):
# class Formatter(logging.Formatter):
# class Logger(logging.Logger):
# 
# Path: memsql_loader/util/webhdfs.py
# def get_webhdfs_url(hdfs_host, webhdfs_port, hdfs_user, op, path):
# 
# Path: memsql_loader/util/attr_dict.py
# class AttrDict(dict):
#     def __getattr__(self, key):
#         try:
#             return self.__getitem__(key)
#         except KeyError:
#             # This lets you use dict-type attributes that aren't keys
#             return getattr(super(AttrDict, self), key)
# 
#     def __setattr__(self, key, value):
#         return self.__setitem__(key, value)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     def __repr__(self):
#         return '%s(%s)' % (self.__class__.__name__, dict.__repr__(self))
# 
#     @staticmethod
#     def from_dict(source):
#         def _transform(d):
#             """ Turns a nested dict into nested AttrDict's """
#             for k, v in d.iteritems():
#                 if isinstance(v, dict):
#                     d[k] = _transform(v)
#             return AttrDict(d)
# 
#         return _transform(source)


import pycurl
import subprocess
import select
import sys
import threading
import time
import os
import zlib
import pywebhdfs.errors
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from memsql_loader.execution.errors import WorkerException, ConnectionException, RequeueTask
from memsql_loader.util import log, webhdfs
from wraptor.decorators import throttle
from memsql_loader.util.attr_dict import AttrDict
from memsql_loader.vendor import glob2
from pywebhdfs.webhdfs import PyWebHdfsClient
        self.job = job
        self.task = task
        self.fifo = fifo
        self.key = None
        self.script_proc = None
        self.decompress_obj = None
        self.pycurl_callback_exception = None

        if task.data['scheme'] == 's3':
            self.is_anonymous = job.spec.source.aws_access_key is None or job.spec.source.aws_secret_key is None
            if self.is_anonymous:
                s3_conn = S3Connection(anon=True)
            else:
                s3_conn = S3Connection(job.spec.source.aws_access_key, job.spec.source.aws_secret_key)
            bucket = s3_conn.get_bucket(task.data['bucket'])

            try:
                self.key = bucket.get_key(task.data['key_name'])
            except S3ResponseError as e:
                raise WorkerException("Received %s %s accessing `%s`, aborting" % (e.status, e.reason, task.data['key_name']))
        elif task.data['scheme'] == 'hdfs':
            fname = task.data['key_name']
            client = PyWebHdfsClient(
                job.spec.source.hdfs_host,
                job.spec.source.webhdfs_port,
                user_name=job.spec.source.hdfs_user)
            try:
                filesize = client.get_file_dir_status(fname)['FileStatus']['length']
            except pywebhdfs.errors.FileNotFound:
                raise WorkerException("File '%s' does not exist on HDFS" % fname)

# Path: memsql_loader/execution/errors.py
# class WorkerException(Exception):
#     def __init__(self, *args, **kwargs):
#         super(Exception, self).__init__(*args, **kwargs)
#         self.time = time.time()
# 
# class ConnectionException(WorkerException):
#     pass
# 
# class RequeueTask(Exception):
#     pass
# 
# Path: memsql_loader/util/log.py
# def setup(log_path=None, stdout_enabled=True):
# def update_verbosity(debug=False, extra_verbose=False):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
# def configure(parser):
# def get_logger(name, level=None):
#     def format(self, record):
#     def __init__(self, name, level=logging.DEBUG):
# class _SetDebug(argparse.Action):
# class _SetStdout(argparse.Action):
# class _SetLogPath(argparse.Action):
# class Formatter(logging.Formatter):
# class Logger(logging.Logger):
# 
# Path: memsql_loader/util/webhdfs.py
# def get_webhdfs_url(hdfs_host, webhdfs_port, hdfs_user, op, path):
# 
# Path: memsql_loader/util/attr_dict.py
# class AttrDict(dict):
#     def __getattr__(self, key):
#         try:
#             return self.__getitem__(key)
#         except KeyError:
#             # This lets you use dict-type attributes that aren't keys
#             return getattr(super(AttrDict, self), key)
# 
#     def __setattr__(self, key, value):
#         return self.__setitem__(key, value)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     def __repr__(self):
#         return '%s(%s)' % (self.__class__.__name__, dict.__repr__(self))
# 
#     @staticmethod
#     def from_dict(source):
#         def _transform(d):
#             """ Turns a nested dict into nested AttrDict's """
#             for k, v in d.iteritems():
#                 if isinstance(v, dict):
#                     d[k] = _transform(v)
#             return AttrDict(d)
# 
#         return _transform(source)


# Path: memsql_loader/execution/downloader.py
import pycurl
import subprocess
import select
import sys
import threading
import time
import os
import zlib
import pywebhdfs.errors
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from memsql_loader.execution.errors import WorkerException, ConnectionException, RequeueTask
from memsql_loader.util import log, webhdfs
from wraptor.decorators import throttle
from memsql_loader.util.attr_dict import AttrDict
from memsql_loader.vendor import glob2
from pywebhdfs.webhdfs import PyWebHdfsClient

        self.job = job
        self.task = task
        self.fifo = fifo
        self.key = None
        self.script_proc = None
        self.decompress_obj = None
        self.pycurl_callback_exception = None

        if task.data['scheme'] == 's3':
            self.is_anonymous = job.spec.source.aws_access_key is None or job.spec.source.aws_secret_key is None
            if self.is_anonymous:
                s3_conn = S3Connection(anon=True)
            else:
                s3_conn = S3Connection(job.spec.source.aws_access_key, job.spec.source.aws_secret_key)
            bucket = s3_conn.get_bucket(task.data['bucket'])

            try:
                self.key = bucket.get_key(task.data['key_name'])
            except S3ResponseError as e:
                raise WorkerException("Received %s %s accessing `%s`, aborting" % (e.status, e.reason, task.data['key_name']))
        elif task.data['scheme'] == 'hdfs':
            fname = task.data['key_name']
            client = PyWebHdfsClient(
                job.spec.source.hdfs_host,
                job.spec.source.webhdfs_port,
                user_name=job.spec.source.hdfs_user)
            try:
                filesize = client.get_file_dir_status(fname)['FileStatus']['length']
            except pywebhdfs.errors.FileNotFound:
                raise WorkerException("File '%s' does not exist on HDFS" % fname)

            self.key = AttrDict({'name': fname, 'size': filesize})