# Path: xpaw/http.py
# class HttpRequest:
#     def __init__(self, url, method="GET", body=None, params=None, headers=None, proxy=None,
#                  timeout=20, verify_ssl=False, allow_redirects=True, auth=None, proxy_auth=None,
#                  priority=None, dont_filter=False, callback=None, errback=None, meta=None,
#                  render=None):
#         """
#         Construct an HTTP request.
#         """
#         self.url = make_url(url, params)
#         self.method = method
#         self.body = body
#         self.headers = headers
#         self.proxy = proxy
#         self.timeout = timeout
#         self.verify_ssl = verify_ssl
#         self.allow_redirects = allow_redirects
#         self.auth = auth
#         self.proxy_auth = proxy_auth
#         self.priority = priority
#         self.dont_filter = dont_filter
#         self.callback = callback
#         self.errback = errback
#         self._meta = dict(meta) if meta else {}
#         self.render = render
# 
#     def __str__(self):
#         return '<{}, {}>'.format(self.method, self.url)
# 
#     __repr__ = __str__
# 
#     @property
#     def meta(self):
#         return self._meta
# 
#     def copy(self):
#         return self.replace()
# 
#     def replace(self, **kwargs):
#         for i in ["url", "method", "body", "headers", "proxy",
#                   "timeout", "verify_ssl", "allow_redirects", "auth", "proxy_auth",
#                   "priority", "dont_filter", "callback", "errback", "meta",
#                   "render"]:
#             kwargs.setdefault(i, getattr(self, i))
#         return type(self)(**kwargs)
# 
#     def to_dict(self):
#         callback = self.callback
#         if inspect.ismethod(callback):
#             callback = callback.__name__
#         errback = self.errback
#         if inspect.ismethod(errback):
#             errback = errback.__name__
#         d = {
#             'url': self.url,
#             'method': self.method,
#             'body': self.body,
#             'headers': self.headers,
#             'proxy': self.proxy,
#             'timeout': self.timeout,
#             'verify_ssl': self.verify_ssl,
#             'allow_redirects': self.allow_redirects,
#             'auth': self.auth,
#             'proxy_auth': self.proxy_auth,
#             'priority': self.priority,
#             'dont_filter': self.dont_filter,
#             'callback': callback,
#             'errback': errback,
#             'meta': self.meta,
#             'render': self.render
#         }
#         return d
# 
#     @classmethod
#     def from_dict(cls, d):
#         return cls(**d)
# 
# Path: xpaw/dupefilter.py
# class HashDupeFilter:
#     def __init__(self):
#         self._hash = set()
# 
#     def is_duplicated(self, request):
#         if request.dont_filter:
#             return False
#         h = request_fingerprint(request)
#         if h in self._hash:
#             log.debug("%s is duplicated", request)
#             return True
#         self._hash.add(h)
#         return False
# 
#     def clear(self):
#         self._hash.clear()
# 
# Path: xpaw/utils.py
# def make_url(url, params=None):
#     args = []
#     if isinstance(params, dict):
#         for k, v in params.items():
#             if isinstance(v, (tuple, list)):
#                 for i in v:
#                     args.append((k, i))
#             else:
#                 args.append((k, v))
#     elif isinstance(params, (tuple, list)):
#         for k, v in params:
#             args.append((k, v))
#     return url_concat(url, args)


from xpaw.http import HttpRequest
from xpaw.dupefilter import HashDupeFilter
from xpaw.utils import make_url
# coding=utf-8



def run_any_dupe_filter(f):
    r_get = HttpRequest("http://example.com")
    r_get_port_80 = HttpRequest("http://example.com:80")
    r_get_port_81 = HttpRequest("http://example.com:81")
    r_get_dont_filter = HttpRequest("http://example.com", dont_filter=True)
    r_get_dir = HttpRequest("http://example.com/")
    r_get_post = HttpRequest("http://example.com/post")
    r_post = HttpRequest("http://example.com/post", "POST")
    r_post_dir = HttpRequest("http://example.com/post/", "POST")
    r_post_data = HttpRequest("http://example.com/post", "POST", body=b'data')
    r_get_param = HttpRequest(make_url("http://example.com/get", params={'k1': 'v1'}))
    r_get_query = HttpRequest("http://example.com/get?k1=v1")

# Path: xpaw/http.py
# class HttpRequest:
#     def __init__(self, url, method="GET", body=None, params=None, headers=None, proxy=None,
#                  timeout=20, verify_ssl=False, allow_redirects=True, auth=None, proxy_auth=None,
#                  priority=None, dont_filter=False, callback=None, errback=None, meta=None,
#                  render=None):
#         """
#         Construct an HTTP request.
#         """
#         self.url = make_url(url, params)
#         self.method = method
#         self.body = body
#         self.headers = headers
#         self.proxy = proxy
#         self.timeout = timeout
#         self.verify_ssl = verify_ssl
#         self.allow_redirects = allow_redirects
#         self.auth = auth
#         self.proxy_auth = proxy_auth
#         self.priority = priority
#         self.dont_filter = dont_filter
#         self.callback = callback
#         self.errback = errback
#         self._meta = dict(meta) if meta else {}
#         self.render = render
# 
#     def __str__(self):
#         return '<{}, {}>'.format(self.method, self.url)
# 
#     __repr__ = __str__
# 
#     @property
#     def meta(self):
#         return self._meta
# 
#     def copy(self):
#         return self.replace()
# 
#     def replace(self, **kwargs):
#         for i in ["url", "method", "body", "headers", "proxy",
#                   "timeout", "verify_ssl", "allow_redirects", "auth", "proxy_auth",
#                   "priority", "dont_filter", "callback", "errback", "meta",
#                   "render"]:
#             kwargs.setdefault(i, getattr(self, i))
#         return type(self)(**kwargs)
# 
#     def to_dict(self):
#         callback = self.callback
#         if inspect.ismethod(callback):
#             callback = callback.__name__
#         errback = self.errback
#         if inspect.ismethod(errback):
#             errback = errback.__name__
#         d = {
#             'url': self.url,
#             'method': self.method,
#             'body': self.body,
#             'headers': self.headers,
#             'proxy': self.proxy,
#             'timeout': self.timeout,
#             'verify_ssl': self.verify_ssl,
#             'allow_redirects': self.allow_redirects,
#             'auth': self.auth,
#             'proxy_auth': self.proxy_auth,
#             'priority': self.priority,
#             'dont_filter': self.dont_filter,
#             'callback': callback,
#             'errback': errback,
#             'meta': self.meta,
#             'render': self.render
#         }
#         return d
# 
#     @classmethod
#     def from_dict(cls, d):
#         return cls(**d)
# 
# Path: xpaw/dupefilter.py
# class HashDupeFilter:
#     def __init__(self):
#         self._hash = set()
# 
#     def is_duplicated(self, request):
#         if request.dont_filter:
#             return False
#         h = request_fingerprint(request)
#         if h in self._hash:
#             log.debug("%s is duplicated", request)
#             return True
#         self._hash.add(h)
#         return False
# 
#     def clear(self):
#         self._hash.clear()
# 
# Path: xpaw/utils.py
# def make_url(url, params=None):
#     args = []
#     if isinstance(params, dict):
#         for k, v in params.items():
#             if isinstance(v, (tuple, list)):
#                 for i in v:
#                     args.append((k, i))
#             else:
#                 args.append((k, v))
#     elif isinstance(params, (tuple, list)):
#         for k, v in params:
#             args.append((k, v))
#     return url_concat(url, args)


# Path: tests/test_dupefilter.py
from xpaw.http import HttpRequest
from xpaw.dupefilter import HashDupeFilter
from xpaw.utils import make_url

# coding=utf-8



def run_any_dupe_filter(f):
    r_get = HttpRequest("http://example.com")
    r_get_port_80 = HttpRequest("http://example.com:80")
    r_get_port_81 = HttpRequest("http://example.com:81")
    r_get_dont_filter = HttpRequest("http://example.com", dont_filter=True)
    r_get_dir = HttpRequest("http://example.com/")
    r_get_post = HttpRequest("http://example.com/post")
    r_post = HttpRequest("http://example.com/post", "POST")
    r_post_dir = HttpRequest("http://example.com/post/", "POST")
    r_post_data = HttpRequest("http://example.com/post", "POST", body=b'data')
    r_get_param = HttpRequest(make_url("http://example.com/get", params={'k1': 'v1'}))
    r_get_query = HttpRequest("http://example.com/get?k1=v1")

    r_get_param_2 = HttpRequest(make_url("http://example.com/get", params={'k1': 'v1', 'k2': 'v2'}))