# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re

    Implementation notes:

    - The current implementation is fault tolerant, meaning it will try
      to extract URLs even if the HTML is malformed and browsers wouldn't
      normally see those links. This may therefore result in some false
      positives.

    - HTML5 tags are supported, including tags not currently supported by
      any major browser.

    :param raw_html: Raw HTML data.
    :type raw_html: str

    :param base_url: Base URL for the current document.
    :type base_url: str

    :param only_links: If True, only extract links to other resources.
        If False, extract all URLs.
    :type only_links: bool

    :returns: Extracted URLs.
    :rtype: set(str)
    """

    # Set where the URLs will be collected.
    result = set()
    add_result = result.add

    # Remove the fragment from the base URL.

# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


# Path: golismero/api/net/scraper.py
from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re


    Implementation notes:

    - The current implementation is fault tolerant, meaning it will try
      to extract URLs even if the HTML is malformed and browsers wouldn't
      normally see those links. This may therefore result in some false
      positives.

    - HTML5 tags are supported, including tags not currently supported by
      any major browser.

    :param raw_html: Raw HTML data.
    :type raw_html: str

    :param base_url: Base URL for the current document.
    :type base_url: str

    :param only_links: If True, only extract links to other resources.
        If False, extract all URLs.
    :type only_links: bool

    :returns: Extracted URLs.
    :rtype: set(str)
    """

    # Set where the URLs will be collected.
    result = set()
    add_result = result.add

    # Remove the fragment from the base URL.

    base_url = urldefrag(base_url)[0]