# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re
    - Unicode URLs are currently not supported.

    :param text: Text.
    :type text: str

    :param base_url: Base URL for the current document.
        If not specified, relative URLs are ignored.
    :type base_url: str

    :param only_links: If True, only extract links to other resources.
        If False, extract all URLs.
    :type only_links: bool

    :returns: Extracted URLs.
    :rtype: set(str)
    """

    # Trivial case.
    if not text:
        return set()

    # Check the type.
    if not isinstance(text, basestring):
        raise TypeError("Expected string, got %r instead" % type(text))

    # Set where the URLs will be collected.
    result = set()

    # Remove the fragment from the base URL.
    if base_url:

# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


# Path: golismero/api/net/scraper.py
from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re

    - Unicode URLs are currently not supported.

    :param text: Text.
    :type text: str

    :param base_url: Base URL for the current document.
        If not specified, relative URLs are ignored.
    :type base_url: str

    :param only_links: If True, only extract links to other resources.
        If False, extract all URLs.
    :type only_links: bool

    :returns: Extracted URLs.
    :rtype: set(str)
    """

    # Trivial case.
    if not text:
        return set()

    # Check the type.
    if not isinstance(text, basestring):
        raise TypeError("Expected string, got %r instead" % type(text))

    # Set where the URLs will be collected.
    result = set()

    # Remove the fragment from the base URL.
    if base_url:

        base_url = urldefrag(base_url)[0]