# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re
            except Exception:
                warn("Unicode URLs not yet supported: %r" % url)
                continue

            # If a base URL was given...
            if base_url:

                # Canonicalize the URL.
                # Discard it on parse error.
                try:
                    url = urljoin(base_url, url.strip())
                except Exception:
                    continue

                # Skip if we've already seen it.
                if url in result:
                    continue

                # Discard URLs that are not links to other pages or resources,
                # and URLs we've already seen.
                if only_links and (url in result or
                                       not is_link(url, base_url = base_url)):
                    continue

            # If a base URL was NOT given...
            else:

                # Discard relative URLs.
                # Also discard them on parse error.
                try:

# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


# Path: golismero/api/net/scraper.py
from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re

            except Exception:
                warn("Unicode URLs not yet supported: %r" % url)
                continue

            # If a base URL was given...
            if base_url:

                # Canonicalize the URL.
                # Discard it on parse error.
                try:
                    url = urljoin(base_url, url.strip())
                except Exception:
                    continue

                # Skip if we've already seen it.
                if url in result:
                    continue

                # Discard URLs that are not links to other pages or resources,
                # and URLs we've already seen.
                if only_links and (url in result or
                                       not is_link(url, base_url = base_url)):
                    continue

            # If a base URL was NOT given...
            else:

                # Discard relative URLs.
                # Also discard them on parse error.
                try:

                    parsed = parse_url(url)