# Path: cocrawler/urls.py
# class URL(object):
#     '''
#     Container for urls and url processing.
#     Precomputes a lot of stuff upon creation, which is usually done in a burner thread.
#     Currently idempotent.
#     '''
#     def __init__(self, url, urljoin=None, surt_strip_trailing_slash=False):
#         url = clean_webpage_links(url, urljoin=urljoin)
# 
#         if urljoin:
#             if isinstance(urljoin, str):
#                 urljoin = URL(urljoin)
#             # optimize a few common cases to dodge full urljoin cost
#             if url.startswith('http://') or url.startswith('https://'):
#                 pass
#             elif url.startswith('/') and not url.startswith('//'):
#                 url = urljoin.urlsplit.scheme + '://' + urljoin.hostname + url
#             else:
#                 url = urllib.parse.urljoin(urljoin.url, url)  # expensive
# 
#         # TODO safe_url_canon has the parsed url, have it pass back the parts
#         url, frag = safe_url_canonicalization(url)
# 
#         if len(frag) > 0:
#             self._original_frag = frag
#         else:
#             self._original_frag = None
# 
#         try:
#             self._urlsplit = urllib.parse.urlsplit(url)  # expensive
#         except ValueError:
#             LOGGER.info('invalid url %s sent into URL constructor', url)
#             # TODO: my code assumes URL() returns something valid, so...
#             # attempt to get rid of anything that would cause an invalid ipv6 ValueError
#             # XXX this isn't the only place that needs tweaking
#             url = url.replace('[', '').replace(']', '')
#             self._urlsplit = urllib.parse.urlsplit(url)
# 
#         (scheme, netloc, path, query, _) = self._urlsplit
# 
#         if path == '':
#             path = '/'
# 
#         # TODO: there's a fair bit of duplicate computing in here
#         netloc = surt.netloc_to_punycanon(scheme, netloc)
#         self._netloc = netloc
#         self._hostname = surt.hostname_to_punycanon(netloc)
#         self._hostname_without_www = surt.discard_www_from_hostname(self._hostname)
#         self._surt = surt.surt(url, surt_strip_trailing_slash=surt_strip_trailing_slash)
# 
#         self._urlsplit = SplitResult(scheme, netloc, path, query, '')
#         self._url = urllib.parse.urlunsplit(self._urlsplit)  # final canonicalization
#         try:
#             # see note above about private domains
#             self._tldextract = tldextract.extract(self._url, include_psl_private_domains=True)
#         except IndexError:
#             # can be raised for punycoded hostnames
#             raise
#         self._registered_domain = self._tldextract.registered_domain
#         if not self._registered_domain:
#             self._registered_domain = self._tldextract.suffix  # example used to be: s3.amazonaws.com, but no longer
# 
#     @property
#     def url(self):
#         return self._url
# 
#     def __str__(self):
#         return self._url
# 
#     @property
#     def urlsplit(self):
#         return self._urlsplit
# 
#     @property
#     def netloc(self):
#         return self._netloc
# 
#     @property
#     def hostname(self):
#         return self._hostname
# 
#     @property
#     def hostname_without_www(self):
#         return self._hostname_without_www
# 
#     @property
#     def surt(self):
#         return self._surt
# 
#     @property
#     def registered_domain(self):
#         return self._registered_domain
# 
#     @property
#     def original_frag(self):
#         return self._original_frag


import sys
import time
import argparse
import asyncio
import random
import os
import cocrawler.dns as dns
import cocrawler.config as config
from cocrawler.urls import URL
resolver = dns.get_resolver()


def create_queue():
    queue = asyncio.Queue()

    # add a fake domain to make sure the dns doesn't send unknown hosts to a search
    # note that mail.foo.com and mx.foo.com don't generally get bogus answers, it's foo.com or www.foo.com that do
    for _ in range(2):
        r = random.Random()
        host = str(r.randrange(1000000000)) + str(r.randrange(1000000000)) + str(r.randrange(1000000000))
        queue.put_nowait((URL('http://' + host + '.com'), 'fake'))

    # read list of domains to query -- from alexa top million
    head, tail = os.path.split(__file__)
    alexa = os.path.join(head, os.pardir, 'data', 'top-1k.txt')
    alexa_count = 0

    try:
        with open(alexa, 'r') as f:
            print('Using top-1k from Alexa, expect a few failures')
            for line in f:
                queue.put_nowait((URL('http://'+line.strip()), 'real'))
                alexa_count += 1
                if alexa_count > args.count:
                    break
    except FileNotFoundError:
        # the alexa file wasn't available (it is not in the repo) so just do a few
        print('Cannot find top-1k file, so all queries are www.google.com')
        for _ in range(args.count):

# Path: cocrawler/urls.py
# class URL(object):
#     '''
#     Container for urls and url processing.
#     Precomputes a lot of stuff upon creation, which is usually done in a burner thread.
#     Currently idempotent.
#     '''
#     def __init__(self, url, urljoin=None, surt_strip_trailing_slash=False):
#         url = clean_webpage_links(url, urljoin=urljoin)
# 
#         if urljoin:
#             if isinstance(urljoin, str):
#                 urljoin = URL(urljoin)
#             # optimize a few common cases to dodge full urljoin cost
#             if url.startswith('http://') or url.startswith('https://'):
#                 pass
#             elif url.startswith('/') and not url.startswith('//'):
#                 url = urljoin.urlsplit.scheme + '://' + urljoin.hostname + url
#             else:
#                 url = urllib.parse.urljoin(urljoin.url, url)  # expensive
# 
#         # TODO safe_url_canon has the parsed url, have it pass back the parts
#         url, frag = safe_url_canonicalization(url)
# 
#         if len(frag) > 0:
#             self._original_frag = frag
#         else:
#             self._original_frag = None
# 
#         try:
#             self._urlsplit = urllib.parse.urlsplit(url)  # expensive
#         except ValueError:
#             LOGGER.info('invalid url %s sent into URL constructor', url)
#             # TODO: my code assumes URL() returns something valid, so...
#             # attempt to get rid of anything that would cause an invalid ipv6 ValueError
#             # XXX this isn't the only place that needs tweaking
#             url = url.replace('[', '').replace(']', '')
#             self._urlsplit = urllib.parse.urlsplit(url)
# 
#         (scheme, netloc, path, query, _) = self._urlsplit
# 
#         if path == '':
#             path = '/'
# 
#         # TODO: there's a fair bit of duplicate computing in here
#         netloc = surt.netloc_to_punycanon(scheme, netloc)
#         self._netloc = netloc
#         self._hostname = surt.hostname_to_punycanon(netloc)
#         self._hostname_without_www = surt.discard_www_from_hostname(self._hostname)
#         self._surt = surt.surt(url, surt_strip_trailing_slash=surt_strip_trailing_slash)
# 
#         self._urlsplit = SplitResult(scheme, netloc, path, query, '')
#         self._url = urllib.parse.urlunsplit(self._urlsplit)  # final canonicalization
#         try:
#             # see note above about private domains
#             self._tldextract = tldextract.extract(self._url, include_psl_private_domains=True)
#         except IndexError:
#             # can be raised for punycoded hostnames
#             raise
#         self._registered_domain = self._tldextract.registered_domain
#         if not self._registered_domain:
#             self._registered_domain = self._tldextract.suffix  # example used to be: s3.amazonaws.com, but no longer
# 
#     @property
#     def url(self):
#         return self._url
# 
#     def __str__(self):
#         return self._url
# 
#     @property
#     def urlsplit(self):
#         return self._urlsplit
# 
#     @property
#     def netloc(self):
#         return self._netloc
# 
#     @property
#     def hostname(self):
#         return self._hostname
# 
#     @property
#     def hostname_without_www(self):
#         return self._hostname_without_www
# 
#     @property
#     def surt(self):
#         return self._surt
# 
#     @property
#     def registered_domain(self):
#         return self._registered_domain
# 
#     @property
#     def original_frag(self):
#         return self._original_frag


# Path: scripts/bench_dns.py
import sys
import time
import argparse
import asyncio
import random
import os
import cocrawler.dns as dns
import cocrawler.config as config
from cocrawler.urls import URL

resolver = dns.get_resolver()


def create_queue():
    queue = asyncio.Queue()

    # add a fake domain to make sure the dns doesn't send unknown hosts to a search
    # note that mail.foo.com and mx.foo.com don't generally get bogus answers, it's foo.com or www.foo.com that do
    for _ in range(2):
        r = random.Random()
        host = str(r.randrange(1000000000)) + str(r.randrange(1000000000)) + str(r.randrange(1000000000))
        queue.put_nowait((URL('http://' + host + '.com'), 'fake'))

    # read list of domains to query -- from alexa top million
    head, tail = os.path.split(__file__)
    alexa = os.path.join(head, os.pardir, 'data', 'top-1k.txt')
    alexa_count = 0

    try:
        with open(alexa, 'r') as f:
            print('Using top-1k from Alexa, expect a few failures')
            for line in f:
                queue.put_nowait((URL('http://'+line.strip()), 'real'))
                alexa_count += 1
                if alexa_count > args.count:
                    break
    except FileNotFoundError:
        # the alexa file wasn't available (it is not in the repo) so just do a few
        print('Cannot find top-1k file, so all queries are www.google.com')
        for _ in range(args.count):

            queue.put_nowait((URL('http://www.google.com'), 'real'))