# Path: memsql_loader/execution/errors.py
# class WorkerException(Exception):
#     def __init__(self, *args, **kwargs):
#         super(Exception, self).__init__(*args, **kwargs)
#         self.time = time.time()
# 
# class ConnectionException(WorkerException):
#     pass
# 
# class RequeueTask(Exception):
#     pass
# 
# Path: memsql_loader/util/log.py
# def setup(log_path=None, stdout_enabled=True):
# def update_verbosity(debug=False, extra_verbose=False):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
# def configure(parser):
# def get_logger(name, level=None):
#     def format(self, record):
#     def __init__(self, name, level=logging.DEBUG):
# class _SetDebug(argparse.Action):
# class _SetStdout(argparse.Action):
# class _SetLogPath(argparse.Action):
# class Formatter(logging.Formatter):
# class Logger(logging.Logger):
# 
# Path: memsql_loader/util/webhdfs.py
# def get_webhdfs_url(hdfs_host, webhdfs_port, hdfs_user, op, path):
# 
# Path: memsql_loader/util/attr_dict.py
# class AttrDict(dict):
#     def __getattr__(self, key):
#         try:
#             return self.__getitem__(key)
#         except KeyError:
#             # This lets you use dict-type attributes that aren't keys
#             return getattr(super(AttrDict, self), key)
# 
#     def __setattr__(self, key, value):
#         return self.__setitem__(key, value)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     def __repr__(self):
#         return '%s(%s)' % (self.__class__.__name__, dict.__repr__(self))
# 
#     @staticmethod
#     def from_dict(source):
#         def _transform(d):
#             """ Turns a nested dict into nested AttrDict's """
#             for k, v in d.iteritems():
#                 if isinstance(v, dict):
#                     d[k] = _transform(v)
#             return AttrDict(d)
# 
#         return _transform(source)


import pycurl
import subprocess
import select
import sys
import threading
import time
import os
import zlib
import pywebhdfs.errors
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from memsql_loader.execution.errors import WorkerException, ConnectionException, RequeueTask
from memsql_loader.util import log, webhdfs
from wraptor.decorators import throttle
from memsql_loader.util.attr_dict import AttrDict
from memsql_loader.vendor import glob2
from pywebhdfs.webhdfs import PyWebHdfsClient

                        # If we're piping data into a script and this file is
                        # a gzipped file, we'll decompress the data ourselves
                        # before piping it into the script.
                        if self.task.data['key_name'].endswith('.gz'):
                            # Set the window bits during decompression to
                            # zlib.MAX_WBITS | 32 tells the zlib library to
                            # automatically detect gzip headers.
                            self.decompress_obj = zlib.decompressobj(zlib.MAX_WBITS | 32)

                        curl.setopt(pycurl.WRITEFUNCTION, self._write_to_fifo(self.script_proc.stdin))
                    else:
                        curl.setopt(pycurl.WRITEFUNCTION, self._write_to_fifo(target_file))

                    if self.task.data['scheme'] == 'hdfs':
                        curl.setopt(pycurl.FOLLOWLOCATION, True)

                    self.logger.info('Starting download')
                    with self.task.protect():
                        self.task.start_step('download')

                    try:
                        curl.perform()
                        status_code = curl.getinfo(pycurl.HTTP_CODE)
                        # HTTP client errors will cause task failure (no retry)
                        if status_code >= 400 and status_code < 500:
                            raise WorkerException('HTTP status code %s for file %s' % (status_code, self.key.name))
                        # HTTP server errors will cause task retry
                        elif status_code >= 500:
                            self.logger.warn('Received HTTP status code %s for file %s, requeueing' % (status_code, self.key.name))

# Path: memsql_loader/execution/errors.py
# class WorkerException(Exception):
#     def __init__(self, *args, **kwargs):
#         super(Exception, self).__init__(*args, **kwargs)
#         self.time = time.time()
# 
# class ConnectionException(WorkerException):
#     pass
# 
# class RequeueTask(Exception):
#     pass
# 
# Path: memsql_loader/util/log.py
# def setup(log_path=None, stdout_enabled=True):
# def update_verbosity(debug=False, extra_verbose=False):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
#     def __call__(self, parser, namespace, value, option_string=None):
# def configure(parser):
# def get_logger(name, level=None):
#     def format(self, record):
#     def __init__(self, name, level=logging.DEBUG):
# class _SetDebug(argparse.Action):
# class _SetStdout(argparse.Action):
# class _SetLogPath(argparse.Action):
# class Formatter(logging.Formatter):
# class Logger(logging.Logger):
# 
# Path: memsql_loader/util/webhdfs.py
# def get_webhdfs_url(hdfs_host, webhdfs_port, hdfs_user, op, path):
# 
# Path: memsql_loader/util/attr_dict.py
# class AttrDict(dict):
#     def __getattr__(self, key):
#         try:
#             return self.__getitem__(key)
#         except KeyError:
#             # This lets you use dict-type attributes that aren't keys
#             return getattr(super(AttrDict, self), key)
# 
#     def __setattr__(self, key, value):
#         return self.__setitem__(key, value)
# 
#     def __str__(self):
#         return self.__repr__()
# 
#     def __repr__(self):
#         return '%s(%s)' % (self.__class__.__name__, dict.__repr__(self))
# 
#     @staticmethod
#     def from_dict(source):
#         def _transform(d):
#             """ Turns a nested dict into nested AttrDict's """
#             for k, v in d.iteritems():
#                 if isinstance(v, dict):
#                     d[k] = _transform(v)
#             return AttrDict(d)
# 
#         return _transform(source)


# Path: memsql_loader/execution/downloader.py
import pycurl
import subprocess
import select
import sys
import threading
import time
import os
import zlib
import pywebhdfs.errors
from boto.exception import S3ResponseError
from boto.s3.connection import S3Connection
from memsql_loader.execution.errors import WorkerException, ConnectionException, RequeueTask
from memsql_loader.util import log, webhdfs
from wraptor.decorators import throttle
from memsql_loader.util.attr_dict import AttrDict
from memsql_loader.vendor import glob2
from pywebhdfs.webhdfs import PyWebHdfsClient


                        # If we're piping data into a script and this file is
                        # a gzipped file, we'll decompress the data ourselves
                        # before piping it into the script.
                        if self.task.data['key_name'].endswith('.gz'):
                            # Set the window bits during decompression to
                            # zlib.MAX_WBITS | 32 tells the zlib library to
                            # automatically detect gzip headers.
                            self.decompress_obj = zlib.decompressobj(zlib.MAX_WBITS | 32)

                        curl.setopt(pycurl.WRITEFUNCTION, self._write_to_fifo(self.script_proc.stdin))
                    else:
                        curl.setopt(pycurl.WRITEFUNCTION, self._write_to_fifo(target_file))

                    if self.task.data['scheme'] == 'hdfs':
                        curl.setopt(pycurl.FOLLOWLOCATION, True)

                    self.logger.info('Starting download')
                    with self.task.protect():
                        self.task.start_step('download')

                    try:
                        curl.perform()
                        status_code = curl.getinfo(pycurl.HTTP_CODE)
                        # HTTP client errors will cause task failure (no retry)
                        if status_code >= 400 and status_code < 500:
                            raise WorkerException('HTTP status code %s for file %s' % (status_code, self.key.name))
                        # HTTP server errors will cause task retry
                        elif status_code >= 500:
                            self.logger.warn('Received HTTP status code %s for file %s, requeueing' % (status_code, self.key.name))

                            raise RequeueTask()