# Path: pyminer/fetcher.py
# class Fetcher(object):
#   """docstring for Fetcher"""
#   def __init__(self, url, headers = {}):
#     super(Fetcher, self).__init__()
#     self.url = url
#     self.headers = headers
# 
#   def perform(self):
#     try:
#       ua = make_ua()
#       ua.update(self.headers)
#       r = requests.get(self.url, headers = ua)
#       r.raise_for_status()
#     except requests.exceptions.HTTPError:
#       r.raise_for_status()
#     except requests.exceptions.RequestException as e:
#       print(e)
# 
#     type = detect_type(r)
#     path = make_path(r, type)
#     write_disk(r, path)
# 
#     return Fetched(self.url, path, type)


from .crutils import *
from .fetcher import Fetcher
    for example lxml

    Usage::

        from pyminer import Miner

        # pdf
        url = "http://www.banglajol.info/index.php/AJMBR/article/viewFile/25509/17126"
        out = miner.fetch(url)
        out.url
        out.path
        out.type
        out.parse()

        # xml
        url = "https://peerj.com/articles/cs-23.xml"
        out = miner.fetch(url)
        out.url
        out.path
        out.type
        out.parse()
        ## or drop down to individual parsing methods
        from pyminer import parsers as p
        p.parse_xml(out.path)
        p.parse_xml_string(out.path)

        # search first, then pass links to fetch
        res = miner.search()
        miner.fetch(res['url'])
    '''

# Path: pyminer/fetcher.py
# class Fetcher(object):
#   """docstring for Fetcher"""
#   def __init__(self, url, headers = {}):
#     super(Fetcher, self).__init__()
#     self.url = url
#     self.headers = headers
# 
#   def perform(self):
#     try:
#       ua = make_ua()
#       ua.update(self.headers)
#       r = requests.get(self.url, headers = ua)
#       r.raise_for_status()
#     except requests.exceptions.HTTPError:
#       r.raise_for_status()
#     except requests.exceptions.RequestException as e:
#       print(e)
# 
#     type = detect_type(r)
#     path = make_path(r, type)
#     write_disk(r, path)
# 
#     return Fetched(self.url, path, type)


# Path: pyminer/fetch.py
from .crutils import *
from .fetcher import Fetcher

    for example lxml

    Usage::

        from pyminer import Miner

        # pdf
        url = "http://www.banglajol.info/index.php/AJMBR/article/viewFile/25509/17126"
        out = miner.fetch(url)
        out.url
        out.path
        out.type
        out.parse()

        # xml
        url = "https://peerj.com/articles/cs-23.xml"
        out = miner.fetch(url)
        out.url
        out.path
        out.type
        out.parse()
        ## or drop down to individual parsing methods
        from pyminer import parsers as p
        p.parse_xml(out.path)
        p.parse_xml_string(out.path)

        # search first, then pass links to fetch
        res = miner.search()
        miner.fetch(res['url'])
    '''

    return Fetcher(url).perform()