# Path: e01extract.py
# def canonicalize(url):
#     parts = list(urlparse(url))
#     if parts[2] == '':
#         parts[2] = '/'  # Empty path equals root path
#     parts[5] = ''       # Erase fragment
#     return urlunparse(parts)
# 
# def same_domain(a, b):
#     parsed_a = urlparse(a)
#     parsed_b = urlparse(b)
#     if parsed_a.netloc == parsed_b.netloc:
#         return True
#     if (parsed_a.netloc == '') ^ (parsed_b.netloc == ''):  # Relative paths
#         return True
#     return False
# 
# URL_EXPR = re.compile(
#     '([a-zA-Z]+\s*=\s*["\'])'   # Tag attribute: href="
#     '(?P<url>'
#         '((http(s?):)?'         # Optional scheme
#         '//[^"\'\s\\\\</]+)?'   # Optional domain
#         '/[^"\'\s\\\\<]*'       # Required path
#     ')')


import logging
import re
import aiohttp
import asyncio
from sys import argv
from urllib.parse import urljoin
from urllib.parse import urlparse
from urllib.parse import urlunparse
from e01extract import canonicalize, same_domain, URL_EXPR

logging.getLogger().setLevel(logging.INFO)



@asyncio.coroutine
def fetch_async(url):
    logging.info('Fetching %s', url)
    response = yield from aiohttp.request('get', url, timeout=5)
    try:
        assert response.status == 200
        data = yield from response.read()
        assert data
        return data.decode('utf-8')
    finally:
        response.close()


@asyncio.coroutine
def extract_async(url):
    data = yield from fetch_async(url)
    found_urls = set()
    for match in URL_EXPR.finditer(data):
        found = canonicalize(match.group('url'))
        if same_domain(url, found):
            found_urls.add(urljoin(url, found))
    return url, data, sorted(found_urls)


def main():

# Path: e01extract.py
# def canonicalize(url):
#     parts = list(urlparse(url))
#     if parts[2] == '':
#         parts[2] = '/'  # Empty path equals root path
#     parts[5] = ''       # Erase fragment
#     return urlunparse(parts)
# 
# def same_domain(a, b):
#     parsed_a = urlparse(a)
#     parsed_b = urlparse(b)
#     if parsed_a.netloc == parsed_b.netloc:
#         return True
#     if (parsed_a.netloc == '') ^ (parsed_b.netloc == ''):  # Relative paths
#         return True
#     return False
# 
# URL_EXPR = re.compile(
#     '([a-zA-Z]+\s*=\s*["\'])'   # Tag attribute: href="
#     '(?P<url>'
#         '((http(s?):)?'         # Optional scheme
#         '//[^"\'\s\\\\</]+)?'   # Optional domain
#         '/[^"\'\s\\\\<]*'       # Required path
#     ')')


# Path: e06asyncextract.py
import logging
import re
import aiohttp
import asyncio
from sys import argv
from urllib.parse import urljoin
from urllib.parse import urlparse
from urllib.parse import urlunparse
from e01extract import canonicalize, same_domain, URL_EXPR


logging.getLogger().setLevel(logging.INFO)



@asyncio.coroutine
def fetch_async(url):
    logging.info('Fetching %s', url)
    response = yield from aiohttp.request('get', url, timeout=5)
    try:
        assert response.status == 200
        data = yield from response.read()
        assert data
        return data.decode('utf-8')
    finally:
        response.close()


@asyncio.coroutine
def extract_async(url):
    data = yield from fetch_async(url)
    found_urls = set()
    for match in URL_EXPR.finditer(data):
        found = canonicalize(match.group('url'))
        if same_domain(url, found):
            found_urls.add(urljoin(url, found))
    return url, data, sorted(found_urls)


def main():

    url = canonicalize(argv[1])