# Path: pyminer/fetched.py
# class Fetched(object):
#   """docstring for Fetched"""
#   def __init__(self, url, path, type):
#     super(Fetched, self).__init__()
#     self.url = url
#     self.path = path
#     self.type = type
# 
#   def parse(self):
#     if self.type == "xml" or self.type == "html":
#       return parse_xml(self.path)
#     elif self.type == "plain":
#       return parse_plain(self.path)
#     elif self.type == "pdf":
#       return parse_pdf(self.path)
#     else:
#       raise Exception("no parsing method for " + self.type)


import requests
from .fetcher_utils import *
from .crutils import *
from .fetched import Fetched

class Fetcher(object):
  """docstring for Fetcher"""
  def __init__(self, url, headers = {}):
    super(Fetcher, self).__init__()
    self.url = url
    self.headers = headers

  def perform(self):
    try:
      ua = make_ua()
      ua.update(self.headers)
      r = requests.get(self.url, headers = ua)
      r.raise_for_status()
    except requests.exceptions.HTTPError:
      r.raise_for_status()
    except requests.exceptions.RequestException as e:
      print(e)

    type = detect_type(r)
    path = make_path(r, type)
    write_disk(r, path)


# Path: pyminer/fetched.py
# class Fetched(object):
#   """docstring for Fetched"""
#   def __init__(self, url, path, type):
#     super(Fetched, self).__init__()
#     self.url = url
#     self.path = path
#     self.type = type
# 
#   def parse(self):
#     if self.type == "xml" or self.type == "html":
#       return parse_xml(self.path)
#     elif self.type == "plain":
#       return parse_plain(self.path)
#     elif self.type == "pdf":
#       return parse_pdf(self.path)
#     else:
#       raise Exception("no parsing method for " + self.type)


# Path: pyminer/fetcher.py
import requests
from .fetcher_utils import *
from .crutils import *
from .fetched import Fetched


class Fetcher(object):
  """docstring for Fetcher"""
  def __init__(self, url, headers = {}):
    super(Fetcher, self).__init__()
    self.url = url
    self.headers = headers

  def perform(self):
    try:
      ua = make_ua()
      ua.update(self.headers)
      r = requests.get(self.url, headers = ua)
      r.raise_for_status()
    except requests.exceptions.HTTPError:
      r.raise_for_status()
    except requests.exceptions.RequestException as e:
      print(e)

    type = detect_type(r)
    path = make_path(r, type)
    write_disk(r, path)


    return Fetched(self.url, path, type)