# Path: awspider/unicodeconverter.py
# def convertToUTF8( s ):
#     s = UnicodeConverter( s ).unicode 
#     if isinstance( s, unicode ):
#         return s.encode("utf-8")
#     else:
#         return None


import urllib
import time
import dateutil.parser
import logging
from twisted.internet.defer import Deferred
from twisted.internet import reactor, ssl
from twisted.web.client import HTTPClientFactory, _parse
from .unicodeconverter import convertToUTF8
from OpenSSL import SSL
           (Default ``None``)
         * *agent* -- User agent to send with request. (Default 
           ``'RequestQueuer'``)
         * *timeout* -- Request timeout, in seconds. (Default ``60``)
         * *cookies* -- Dictionary of strings to send as request cookies. 
           (Default ``None``).
         * *follow_redirect* -- Boolean switch to follow HTTP redirects. 
           (Default ``True``)
         * *prioritize* -- Move this request to the front of the request 
           queue. (Default ``False``)         

        """
        if headers is None:
            headers={}
        if postdata is not None:
            if isinstance(postdata, dict):
                for key in postdata:
                    postdata[key] = convertToUTF8(postdata[key])
                postdata = urllib.urlencode(postdata)
            else:
                convertToUTF8(postdata)
        if method.lower() == "post":
            headers["content-type"] = "application/x-www-form-urlencoded"
        if last_modified is not None:
            time_tuple = dateutil.parser.parse(last_modified).timetuple()
            time_string = time.strftime("%a, %d %b %Y %T %z", time_tuple)
            headers['If-Modified-Since'] = time_string
        if etag is not None:
            headers["If-None-Match"] = etag
        req = {

# Path: awspider/unicodeconverter.py
# def convertToUTF8( s ):
#     s = UnicodeConverter( s ).unicode 
#     if isinstance( s, unicode ):
#         return s.encode("utf-8")
#     else:
#         return None


# Path: awspider/requestqueuer.py
import urllib
import time
import dateutil.parser
import logging
from twisted.internet.defer import Deferred
from twisted.internet import reactor, ssl
from twisted.web.client import HTTPClientFactory, _parse
from .unicodeconverter import convertToUTF8
from OpenSSL import SSL

           (Default ``None``)
         * *agent* -- User agent to send with request. (Default 
           ``'RequestQueuer'``)
         * *timeout* -- Request timeout, in seconds. (Default ``60``)
         * *cookies* -- Dictionary of strings to send as request cookies. 
           (Default ``None``).
         * *follow_redirect* -- Boolean switch to follow HTTP redirects. 
           (Default ``True``)
         * *prioritize* -- Move this request to the front of the request 
           queue. (Default ``False``)         

        """
        if headers is None:
            headers={}
        if postdata is not None:
            if isinstance(postdata, dict):
                for key in postdata:
                    postdata[key] = convertToUTF8(postdata[key])
                postdata = urllib.urlencode(postdata)
            else:
                convertToUTF8(postdata)
        if method.lower() == "post":
            headers["content-type"] = "application/x-www-form-urlencoded"
        if last_modified is not None:
            time_tuple = dateutil.parser.parse(last_modified).timetuple()
            time_string = time.strftime("%a, %d %b %Y %T %z", time_tuple)
            headers['If-Modified-Since'] = time_string
        if etag is not None:
            headers["If-None-Match"] = etag
        req = {

            "url":convertToUTF8(url),