# Path: jd/proxymanager.py
# class ProxyManager(object):
#     def __init__(self):
#     def update_proxy(self, count = 100):
#     def push_proxy(self, proxy):
#     def get_proxy(self):
#     def delete_proxy(self, proxy):


import logging
from twisted.internet import defer
from twisted.internet.error import TimeoutError, DNSLookupError, \
    ConnectionRefusedError, ConnectionDone, ConnectError, \
    ConnectionLost, TCPTimedOutError
from scrapy.exceptions import NotConfigured
from scrapy.utils.response import response_status_message
from scrapy.xlib.tx import ResponseFailed
from scrapy.core.downloader.handlers.http11 import TunnelError
from jd.proxymanager import proxymng
# coding=utf-8




logger = logging.getLogger(__name__)


class ProxyMiddleware(object):
    def process_request(self, request, spider):
        try:
            request.meta['req_count'] = request.meta.get('req_count', 0) + 1

            if request.meta.get('is_proxy', False):
                request.meta['proxy'] = proxymng.get_proxy()
        except Exception, e:
            logging.warning('ProxyMiddleware Exception:%s' % str(e))

    def process_exception(self, request, exception, spider):
        logging.error('process_exception error_request request exception:%s url:%s  proxy:%s' % (
            exception, request.url, str(request.meta)))

        if request.meta.get('is_proxy', False):

# Path: jd/proxymanager.py
# class ProxyManager(object):
#     def __init__(self):
#     def update_proxy(self, count = 100):
#     def push_proxy(self, proxy):
#     def get_proxy(self):
#     def delete_proxy(self, proxy):


# Path: jd/middlewares/middlewares.py
import logging
from twisted.internet import defer
from twisted.internet.error import TimeoutError, DNSLookupError, \
    ConnectionRefusedError, ConnectionDone, ConnectError, \
    ConnectionLost, TCPTimedOutError
from scrapy.exceptions import NotConfigured
from scrapy.utils.response import response_status_message
from scrapy.xlib.tx import ResponseFailed
from scrapy.core.downloader.handlers.http11 import TunnelError
from jd.proxymanager import proxymng

# coding=utf-8




logger = logging.getLogger(__name__)


class ProxyMiddleware(object):
    def process_request(self, request, spider):
        try:
            request.meta['req_count'] = request.meta.get('req_count', 0) + 1

            if request.meta.get('is_proxy', False):
                request.meta['proxy'] = proxymng.get_proxy()
        except Exception, e:
            logging.warning('ProxyMiddleware Exception:%s' % str(e))

    def process_exception(self, request, exception, spider):
        logging.error('process_exception error_request request exception:%s url:%s  proxy:%s' % (
            exception, request.url, str(request.meta)))

        if request.meta.get('is_proxy', False):

            proxymng.delete_proxy(request.meta.get('proxy'))