# Path: xpaw/utils.py
# def request_fingerprint(request):
#     sha1 = hashlib.sha1()
#     sha1.update(to_bytes(request.method))
#     res = urlsplit(request.url)
#     queries = parse_qsl(res.query)
#     queries.sort()
#     final_query = urlencode(queries)
#     sha1.update(to_bytes('{}://{}{}:{}?{}'.format(res.scheme,
#                                                   '' if res.hostname is None else res.hostname,
#                                                   res.path,
#                                                   80 if res.port is None else res.port,
#                                                   final_query)))
#     sha1.update(request.body or b'')
#     return sha1.hexdigest()


import logging
from .utils import request_fingerprint
# coding=utf-8



log = logging.getLogger(__name__)


class HashDupeFilter:
    def __init__(self):
        self._hash = set()

    def is_duplicated(self, request):
        if request.dont_filter:
            return False
        h = request_fingerprint(request)
        if h in self._hash:
            log.debug("%s is duplicated", request)
            return True
        self._hash.add(h)
        return False

    def clear(self):

# Path: xpaw/utils.py
# def request_fingerprint(request):
#     sha1 = hashlib.sha1()
#     sha1.update(to_bytes(request.method))
#     res = urlsplit(request.url)
#     queries = parse_qsl(res.query)
#     queries.sort()
#     final_query = urlencode(queries)
#     sha1.update(to_bytes('{}://{}{}:{}?{}'.format(res.scheme,
#                                                   '' if res.hostname is None else res.hostname,
#                                                   res.path,
#                                                   80 if res.port is None else res.port,
#                                                   final_query)))
#     sha1.update(request.body or b'')
#     return sha1.hexdigest()


# Path: xpaw/dupefilter.py
import logging
from .utils import request_fingerprint

# coding=utf-8



log = logging.getLogger(__name__)


class HashDupeFilter:
    def __init__(self):
        self._hash = set()

    def is_duplicated(self, request):
        if request.dont_filter:
            return False
        h = request_fingerprint(request)
        if h in self._hash:
            log.debug("%s is duplicated", request)
            return True
        self._hash.add(h)
        return False

    def clear(self):

        self._hash.clear()