# Path: bidwire/scrapers/base_scraper.py
# class BaseScraper(metaclass=abc.ABCMeta):
#     """The interface that must be implemented by all scrapers."""
# 
#     @abc.abstractmethod
#     def scrape(self, session):
#         """
#         This method performs all scraping for this scraper. Must be implemented
#         by subclasses.
# 
#         Arguments:
#         session -- database session to use for persisting items
#         """
#         pass
# 
# Path: bidwire/scrapers/massgov/url_scraper_dict.py
# UL_CATEGORY_LI = '//ul[@class="category"]/li'
# H2_A_TITLELINK = './h2/a[@class="titlelink"]'
# SPAN_A_TITLELINK = './span/a[@class="titlelink"]'
# DIV_BODYFIELD_P = '//div[contains(@class,"bodyfield")]/p'
# CATEGORY_H2_XPATH = [ UL_CATEGORY_LI, H2_A_TITLELINK ]
# BODYFIELD_SPAN_XPATH = [ DIV_BODYFIELD_P, SPAN_A_TITLELINK ]
# MASSGOV_DICT = {
#         'homeland-sec/grants/docs/':
#             [
#                 UL_CATEGORY_LI,
#                 './h2/span/a[@class="titlelink"]'
#             ],
#         'homeland-sec/grants/hs-grant-guidance-and-policies.html':
#             BODYFIELD_SPAN_XPATH,
#         'homeland-sec/grants/standard-documents.html':
#             [
#                 '//div[contains(@class,"bodyfield")]/ul/li',
#                 SPAN_A_TITLELINK
#             ],
#         'law-enforce/grants/': CATEGORY_H2_XPATH,
#         'law-enforce/grants/2017-muni-public-safety-staffing-grant.html':
#             BODYFIELD_SPAN_XPATH,
#         'law-enforce/grants/le-grants-public-records.html':
#             BODYFIELD_SPAN_XPATH,
#         'justice-and-prev/grants/': CATEGORY_H2_XPATH,
#         'justice-and-prev/grants/bgp/': CATEGORY_H2_XPATH,
#         'hwy-safety/grants/': CATEGORY_H2_XPATH,
#         'hwy-safety/grants/ffy-2017-traffic-enforcement-grant-program.html':
#             BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/ffy2017-hsd-grant-opportunities.html':
#             BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/ffy-2017-step.html': BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/highway-safety-grants-public-records.html':
#             BODYFIELD_SPAN_XPATH
#     }
# 
# Path: bidwire/scrapers/massgov/results_page_scraper.py
# SITE_ROOT = 'https://www.mass.gov'
# def scrape_results_page(page_str, xpath_list):


from document import Document, get_new_urls
from .base_scraper import BaseScraper
from .massgov import url_scraper_dict
from .massgov import results_page_scraper
import logging
import scrapelib



log = logging.getLogger(__name__)

URL_PREFIX = 'http://www.mass.gov/eopss/funding-and-training/'


class MassGovEOPSSScraper(BaseScraper):
    def __init__(self):
        self.url_dict = url_scraper_dict.MASSGOV_DICT

    def get_site(self):
        return Document.Site.MASSGOV_EOPSS

    def scrape(self, session):
        """Iterates through all the sites in url_dict to extract new documents.

        This is implemented as follows:
          1. Download each of the pages.
          2. Extract the URLs from the pages.
          3. Check which of those URLs are not yet in our database.
          4. For each of the URLs that are not yet in our database, 
             add them as a new Bid object to the database.
        """
        scraper = scrapelib.Scraper()
        for url, xpaths in self.url_dict.items():
            page = scraper.get(URL_PREFIX + url)
            # doc_ids is dictionary: relative URL => title of doc
            doc_ids = \

# Path: bidwire/scrapers/base_scraper.py
# class BaseScraper(metaclass=abc.ABCMeta):
#     """The interface that must be implemented by all scrapers."""
# 
#     @abc.abstractmethod
#     def scrape(self, session):
#         """
#         This method performs all scraping for this scraper. Must be implemented
#         by subclasses.
# 
#         Arguments:
#         session -- database session to use for persisting items
#         """
#         pass
# 
# Path: bidwire/scrapers/massgov/url_scraper_dict.py
# UL_CATEGORY_LI = '//ul[@class="category"]/li'
# H2_A_TITLELINK = './h2/a[@class="titlelink"]'
# SPAN_A_TITLELINK = './span/a[@class="titlelink"]'
# DIV_BODYFIELD_P = '//div[contains(@class,"bodyfield")]/p'
# CATEGORY_H2_XPATH = [ UL_CATEGORY_LI, H2_A_TITLELINK ]
# BODYFIELD_SPAN_XPATH = [ DIV_BODYFIELD_P, SPAN_A_TITLELINK ]
# MASSGOV_DICT = {
#         'homeland-sec/grants/docs/':
#             [
#                 UL_CATEGORY_LI,
#                 './h2/span/a[@class="titlelink"]'
#             ],
#         'homeland-sec/grants/hs-grant-guidance-and-policies.html':
#             BODYFIELD_SPAN_XPATH,
#         'homeland-sec/grants/standard-documents.html':
#             [
#                 '//div[contains(@class,"bodyfield")]/ul/li',
#                 SPAN_A_TITLELINK
#             ],
#         'law-enforce/grants/': CATEGORY_H2_XPATH,
#         'law-enforce/grants/2017-muni-public-safety-staffing-grant.html':
#             BODYFIELD_SPAN_XPATH,
#         'law-enforce/grants/le-grants-public-records.html':
#             BODYFIELD_SPAN_XPATH,
#         'justice-and-prev/grants/': CATEGORY_H2_XPATH,
#         'justice-and-prev/grants/bgp/': CATEGORY_H2_XPATH,
#         'hwy-safety/grants/': CATEGORY_H2_XPATH,
#         'hwy-safety/grants/ffy-2017-traffic-enforcement-grant-program.html':
#             BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/ffy2017-hsd-grant-opportunities.html':
#             BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/ffy-2017-step.html': BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/highway-safety-grants-public-records.html':
#             BODYFIELD_SPAN_XPATH
#     }
# 
# Path: bidwire/scrapers/massgov/results_page_scraper.py
# SITE_ROOT = 'https://www.mass.gov'
# def scrape_results_page(page_str, xpath_list):


# Path: bidwire/scrapers/massgov_eopss_scraper.py
from document import Document, get_new_urls
from .base_scraper import BaseScraper
from .massgov import url_scraper_dict
from .massgov import results_page_scraper
import logging
import scrapelib




log = logging.getLogger(__name__)

URL_PREFIX = 'http://www.mass.gov/eopss/funding-and-training/'


class MassGovEOPSSScraper(BaseScraper):
    def __init__(self):
        self.url_dict = url_scraper_dict.MASSGOV_DICT

    def get_site(self):
        return Document.Site.MASSGOV_EOPSS

    def scrape(self, session):
        """Iterates through all the sites in url_dict to extract new documents.

        This is implemented as follows:
          1. Download each of the pages.
          2. Extract the URLs from the pages.
          3. Check which of those URLs are not yet in our database.
          4. For each of the URLs that are not yet in our database, 
             add them as a new Bid object to the database.
        """
        scraper = scrapelib.Scraper()
        for url, xpaths in self.url_dict.items():
            page = scraper.get(URL_PREFIX + url)
            # doc_ids is dictionary: relative URL => title of doc
            doc_ids = \

                results_page_scraper.scrape_results_page(page.content, xpaths)