# Path: cocrawler/urls.py
# class URL(object):
#     '''
#     Container for urls and url processing.
#     Precomputes a lot of stuff upon creation, which is usually done in a burner thread.
#     Currently idempotent.
#     '''
#     def __init__(self, url, urljoin=None, surt_strip_trailing_slash=False):
#         url = clean_webpage_links(url, urljoin=urljoin)
# 
#         if urljoin:
#             if isinstance(urljoin, str):
#                 urljoin = URL(urljoin)
#             # optimize a few common cases to dodge full urljoin cost
#             if url.startswith('http://') or url.startswith('https://'):
#                 pass
#             elif url.startswith('/') and not url.startswith('//'):
#                 url = urljoin.urlsplit.scheme + '://' + urljoin.hostname + url
#             else:
#                 url = urllib.parse.urljoin(urljoin.url, url)  # expensive
# 
#         # TODO safe_url_canon has the parsed url, have it pass back the parts
#         url, frag = safe_url_canonicalization(url)
# 
#         if len(frag) > 0:
#             self._original_frag = frag
#         else:
#             self._original_frag = None
# 
#         try:
#             self._urlsplit = urllib.parse.urlsplit(url)  # expensive
#         except ValueError:
#             LOGGER.info('invalid url %s sent into URL constructor', url)
#             # TODO: my code assumes URL() returns something valid, so...
#             # attempt to get rid of anything that would cause an invalid ipv6 ValueError
#             # XXX this isn't the only place that needs tweaking
#             url = url.replace('[', '').replace(']', '')
#             self._urlsplit = urllib.parse.urlsplit(url)
# 
#         (scheme, netloc, path, query, _) = self._urlsplit
# 
#         if path == '':
#             path = '/'
# 
#         # TODO: there's a fair bit of duplicate computing in here
#         netloc = surt.netloc_to_punycanon(scheme, netloc)
#         self._netloc = netloc
#         self._hostname = surt.hostname_to_punycanon(netloc)
#         self._hostname_without_www = surt.discard_www_from_hostname(self._hostname)
#         self._surt = surt.surt(url, surt_strip_trailing_slash=surt_strip_trailing_slash)
# 
#         self._urlsplit = SplitResult(scheme, netloc, path, query, '')
#         self._url = urllib.parse.urlunsplit(self._urlsplit)  # final canonicalization
#         try:
#             # see note above about private domains
#             self._tldextract = tldextract.extract(self._url, include_psl_private_domains=True)
#         except IndexError:
#             # can be raised for punycoded hostnames
#             raise
#         self._registered_domain = self._tldextract.registered_domain
#         if not self._registered_domain:
#             self._registered_domain = self._tldextract.suffix  # example used to be: s3.amazonaws.com, but no longer
# 
#     @property
#     def url(self):
#         return self._url
# 
#     def __str__(self):
#         return self._url
# 
#     @property
#     def urlsplit(self):
#         return self._urlsplit
# 
#     @property
#     def netloc(self):
#         return self._netloc
# 
#     @property
#     def hostname(self):
#         return self._hostname
# 
#     @property
#     def hostname_without_www(self):
#         return self._hostname_without_www
# 
#     @property
#     def surt(self):
#         return self._surt
# 
#     @property
#     def registered_domain(self):
#         return self._registered_domain
# 
#     @property
#     def original_frag(self):
#         return self._original_frag


import tempfile
import os
import pytest
import cocrawler.datalayer as datalayer
import cocrawler.config as config
from cocrawler.urls import URL


def test_seen():
    c = {'Robots': {'RobotsCacheSize': 1, 'RobotsCacheTimeout': 1}}
    config.set_config(c)
    dl = datalayer.Datalayer()
    assert not dl.seen(URL('http://example.com'))
    dl.add_seen(URL('http://example.com'))

# Path: cocrawler/urls.py
# class URL(object):
#     '''
#     Container for urls and url processing.
#     Precomputes a lot of stuff upon creation, which is usually done in a burner thread.
#     Currently idempotent.
#     '''
#     def __init__(self, url, urljoin=None, surt_strip_trailing_slash=False):
#         url = clean_webpage_links(url, urljoin=urljoin)
# 
#         if urljoin:
#             if isinstance(urljoin, str):
#                 urljoin = URL(urljoin)
#             # optimize a few common cases to dodge full urljoin cost
#             if url.startswith('http://') or url.startswith('https://'):
#                 pass
#             elif url.startswith('/') and not url.startswith('//'):
#                 url = urljoin.urlsplit.scheme + '://' + urljoin.hostname + url
#             else:
#                 url = urllib.parse.urljoin(urljoin.url, url)  # expensive
# 
#         # TODO safe_url_canon has the parsed url, have it pass back the parts
#         url, frag = safe_url_canonicalization(url)
# 
#         if len(frag) > 0:
#             self._original_frag = frag
#         else:
#             self._original_frag = None
# 
#         try:
#             self._urlsplit = urllib.parse.urlsplit(url)  # expensive
#         except ValueError:
#             LOGGER.info('invalid url %s sent into URL constructor', url)
#             # TODO: my code assumes URL() returns something valid, so...
#             # attempt to get rid of anything that would cause an invalid ipv6 ValueError
#             # XXX this isn't the only place that needs tweaking
#             url = url.replace('[', '').replace(']', '')
#             self._urlsplit = urllib.parse.urlsplit(url)
# 
#         (scheme, netloc, path, query, _) = self._urlsplit
# 
#         if path == '':
#             path = '/'
# 
#         # TODO: there's a fair bit of duplicate computing in here
#         netloc = surt.netloc_to_punycanon(scheme, netloc)
#         self._netloc = netloc
#         self._hostname = surt.hostname_to_punycanon(netloc)
#         self._hostname_without_www = surt.discard_www_from_hostname(self._hostname)
#         self._surt = surt.surt(url, surt_strip_trailing_slash=surt_strip_trailing_slash)
# 
#         self._urlsplit = SplitResult(scheme, netloc, path, query, '')
#         self._url = urllib.parse.urlunsplit(self._urlsplit)  # final canonicalization
#         try:
#             # see note above about private domains
#             self._tldextract = tldextract.extract(self._url, include_psl_private_domains=True)
#         except IndexError:
#             # can be raised for punycoded hostnames
#             raise
#         self._registered_domain = self._tldextract.registered_domain
#         if not self._registered_domain:
#             self._registered_domain = self._tldextract.suffix  # example used to be: s3.amazonaws.com, but no longer
# 
#     @property
#     def url(self):
#         return self._url
# 
#     def __str__(self):
#         return self._url
# 
#     @property
#     def urlsplit(self):
#         return self._urlsplit
# 
#     @property
#     def netloc(self):
#         return self._netloc
# 
#     @property
#     def hostname(self):
#         return self._hostname
# 
#     @property
#     def hostname_without_www(self):
#         return self._hostname_without_www
# 
#     @property
#     def surt(self):
#         return self._surt
# 
#     @property
#     def registered_domain(self):
#         return self._registered_domain
# 
#     @property
#     def original_frag(self):
#         return self._original_frag


# Path: tests/unit/test_datalayer.py
import tempfile
import os
import pytest
import cocrawler.datalayer as datalayer
import cocrawler.config as config
from cocrawler.urls import URL



def test_seen():
    c = {'Robots': {'RobotsCacheSize': 1, 'RobotsCacheTimeout': 1}}
    config.set_config(c)
    dl = datalayer.Datalayer()
    assert not dl.seen(URL('http://example.com'))
    dl.add_seen(URL('http://example.com'))

    assert dl.seen(URL('http://example.com'))