# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re
            name = tag.get("name", "").lower().strip()
            if name in param_names:
                url = tag.get("value", None)
        ##elif name == "form":
        ##    url = tag.get("action", None)
        elif name == "object":
            url = tag.get("data", None)
        elif name == "applet":
            url = tag.get("code", None)
        elif name == "meta":
            name = tag.get("name", "").lower().strip()
            if name == "http-equiv":
                content = tag.get("content", "")
                p = content.find(";")
                if p >= 0:
                    url = content[ p + 1 : ]
        elif name == "base":
            url = tag.get("href", None)
            if url is not None:

                # XXX FIXME
                # Unicode URLs are not supported.
                try:
                    url = str(url)
                except Exception:
                    warn("Unicode URLs not yet supported: %r" % url)
                    continue

                # Update the base URL.
                try:

# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


# Path: golismero/api/net/scraper.py
from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re

            name = tag.get("name", "").lower().strip()
            if name in param_names:
                url = tag.get("value", None)
        ##elif name == "form":
        ##    url = tag.get("action", None)
        elif name == "object":
            url = tag.get("data", None)
        elif name == "applet":
            url = tag.get("code", None)
        elif name == "meta":
            name = tag.get("name", "").lower().strip()
            if name == "http-equiv":
                content = tag.get("content", "")
                p = content.find(";")
                if p >= 0:
                    url = content[ p + 1 : ]
        elif name == "base":
            url = tag.get("href", None)
            if url is not None:

                # XXX FIXME
                # Unicode URLs are not supported.
                try:
                    url = str(url)
                except Exception:
                    warn("Unicode URLs not yet supported: %r" % url)
                    continue

                # Update the base URL.
                try:

                    base_url = urljoin(base_url, url.strip(),