# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re
#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
URL scraping API.

This module contains utility functions to extract (scrape) URLs from data.
Currently only HTML and plain text data are supported.
"""

__license__ = """
GoLismero 2.0 - The web knife - Copyright (C) 2011-2014

Golismero project site: https://github.com/golismero
Golismero project mail: contact@golismero-project.com

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of

# Path: golismero/api/net/web_utils.py
# def parse_url(url, base_url = None):
#     """
#     Parse an URL and return a mutable object with all its parts.
# 
#     For more details see: ParsedURL
# 
#     :param url: URL to parse.
#     :type url: str
# 
#     :param base_url: Optional base URL.
#     :type base_url: str
# 
#     :returns: Mutable object with access to the URL parts.
#     :rtype: ParsedURL
#     """
#     return ParsedURL(url, base_url)
# 
# def urldefrag(url):
#     p = parse_url(url)
#     f = p.fragment
#     p.fragment = ""
#     return p.url, f
# 
# def urljoin(base_url, url, allow_fragments = True):
#     if not allow_fragments:
#         url = urldefrag(url)
#         base_url = urldefrag(base_url)
#     return parse_url(url, base_url).url


# Path: golismero/api/net/scraper.py
from .web_utils import parse_url, urldefrag, urljoin
from BeautifulSoup import BeautifulSoup
from warnings import warn
from codecs import decode
from chardet import detect
import re

#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
URL scraping API.

This module contains utility functions to extract (scrape) URLs from data.
Currently only HTML and plain text data are supported.
"""

__license__ = """
GoLismero 2.0 - The web knife - Copyright (C) 2011-2014

Golismero project site: https://github.com/golismero
Golismero project mail: contact@golismero-project.com

This program is free software; you can redistribute it and/or
modify it under the terms of the GNU General Public License
as published by the Free Software Foundation; either version 2
of the License, or (at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of

MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the