# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re
# URL detection regex, by John Gruber.
# http://daringfireball.net/2010/07/improved_regex_for_matching_urls
_re_url_readable = re.compile(r"""(?i)\b((?:[a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’]))""", re.I)


#------------------------------------------------------------------------------
# Wrappers for URIs in plain text
# http://www.w3.org/Addressing/URL/url-spec.txt
_re_url_rfc = re.compile(r"""\\<([^\\>]+\\:\\/\\/[^\\>]+)\\>""", re.I)


#------------------------------------------------------------------------------
def is_link(url, base_url):
    """
    Determines if an URL is a link to another resource.

    :param url: URL to test.
    :type url: str

    :param base_url: Base URL for the current document.
        Must not contain a fragment.
    :type base_url: str

    :returns: True if the URL points to another page or resource,
        False otherwise.
    :rtype: bool
    """
    try:

        # Parse the URL. If it can't be parsed, it's not a link.

# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


# Path: golismero/api/net/scraper.py
from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re

# URL detection regex, by John Gruber.
# http://daringfireball.net/2010/07/improved_regex_for_matching_urls
_re_url_readable = re.compile(r"""(?i)\b((?:[a-z][\w-]+:(?:/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’]))""", re.I)


#------------------------------------------------------------------------------
# Wrappers for URIs in plain text
# http://www.w3.org/Addressing/URL/url-spec.txt
_re_url_rfc = re.compile(r"""\\<([^\\>]+\\:\\/\\/[^\\>]+)\\>""", re.I)


#------------------------------------------------------------------------------
def is_link(url, base_url):
    """
    Determines if an URL is a link to another resource.

    :param url: URL to test.
    :type url: str

    :param base_url: Base URL for the current document.
        Must not contain a fragment.
    :type base_url: str

    :returns: True if the URL points to another page or resource,
        False otherwise.
    :rtype: bool
    """
    try:

        # Parse the URL. If it can't be parsed, it's not a link.

        parsed = parse_url(url, base_url)