# Path: bidwire/scrapers/base_scraper.py
# class BaseScraper(metaclass=abc.ABCMeta):
#     """The interface that must be implemented by all scrapers."""
# 
#     @abc.abstractmethod
#     def scrape(self, session):
#         """
#         This method performs all scraping for this scraper. Must be implemented
#         by subclasses.
# 
#         Arguments:
#         session -- database session to use for persisting items
#         """
#         pass
# 
# Path: bidwire/scrapers/massgov/url_scraper_dict.py
# UL_CATEGORY_LI = '//ul[@class="category"]/li'
# H2_A_TITLELINK = './h2/a[@class="titlelink"]'
# SPAN_A_TITLELINK = './span/a[@class="titlelink"]'
# DIV_BODYFIELD_P = '//div[contains(@class,"bodyfield")]/p'
# CATEGORY_H2_XPATH = [ UL_CATEGORY_LI, H2_A_TITLELINK ]
# BODYFIELD_SPAN_XPATH = [ DIV_BODYFIELD_P, SPAN_A_TITLELINK ]
# MASSGOV_DICT = {
#         'homeland-sec/grants/docs/':
#             [
#                 UL_CATEGORY_LI,
#                 './h2/span/a[@class="titlelink"]'
#             ],
#         'homeland-sec/grants/hs-grant-guidance-and-policies.html':
#             BODYFIELD_SPAN_XPATH,
#         'homeland-sec/grants/standard-documents.html':
#             [
#                 '//div[contains(@class,"bodyfield")]/ul/li',
#                 SPAN_A_TITLELINK
#             ],
#         'law-enforce/grants/': CATEGORY_H2_XPATH,
#         'law-enforce/grants/2017-muni-public-safety-staffing-grant.html':
#             BODYFIELD_SPAN_XPATH,
#         'law-enforce/grants/le-grants-public-records.html':
#             BODYFIELD_SPAN_XPATH,
#         'justice-and-prev/grants/': CATEGORY_H2_XPATH,
#         'justice-and-prev/grants/bgp/': CATEGORY_H2_XPATH,
#         'hwy-safety/grants/': CATEGORY_H2_XPATH,
#         'hwy-safety/grants/ffy-2017-traffic-enforcement-grant-program.html':
#             BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/ffy2017-hsd-grant-opportunities.html':
#             BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/ffy-2017-step.html': BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/highway-safety-grants-public-records.html':
#             BODYFIELD_SPAN_XPATH
#     }
# 
# Path: bidwire/scrapers/massgov/results_page_scraper.py
# SITE_ROOT = 'https://www.mass.gov'
# def scrape_results_page(page_str, xpath_list):


from document import Document, get_new_urls
from .base_scraper import BaseScraper
from .massgov import url_scraper_dict
from .massgov import results_page_scraper
import logging
import scrapelib



log = logging.getLogger(__name__)

URL_PREFIX = 'http://www.mass.gov/eopss/funding-and-training/'


class MassGovEOPSSScraper(BaseScraper):
    def __init__(self):
        self.url_dict = url_scraper_dict.MASSGOV_DICT


# Path: bidwire/scrapers/base_scraper.py
# class BaseScraper(metaclass=abc.ABCMeta):
#     """The interface that must be implemented by all scrapers."""
# 
#     @abc.abstractmethod
#     def scrape(self, session):
#         """
#         This method performs all scraping for this scraper. Must be implemented
#         by subclasses.
# 
#         Arguments:
#         session -- database session to use for persisting items
#         """
#         pass
# 
# Path: bidwire/scrapers/massgov/url_scraper_dict.py
# UL_CATEGORY_LI = '//ul[@class="category"]/li'
# H2_A_TITLELINK = './h2/a[@class="titlelink"]'
# SPAN_A_TITLELINK = './span/a[@class="titlelink"]'
# DIV_BODYFIELD_P = '//div[contains(@class,"bodyfield")]/p'
# CATEGORY_H2_XPATH = [ UL_CATEGORY_LI, H2_A_TITLELINK ]
# BODYFIELD_SPAN_XPATH = [ DIV_BODYFIELD_P, SPAN_A_TITLELINK ]
# MASSGOV_DICT = {
#         'homeland-sec/grants/docs/':
#             [
#                 UL_CATEGORY_LI,
#                 './h2/span/a[@class="titlelink"]'
#             ],
#         'homeland-sec/grants/hs-grant-guidance-and-policies.html':
#             BODYFIELD_SPAN_XPATH,
#         'homeland-sec/grants/standard-documents.html':
#             [
#                 '//div[contains(@class,"bodyfield")]/ul/li',
#                 SPAN_A_TITLELINK
#             ],
#         'law-enforce/grants/': CATEGORY_H2_XPATH,
#         'law-enforce/grants/2017-muni-public-safety-staffing-grant.html':
#             BODYFIELD_SPAN_XPATH,
#         'law-enforce/grants/le-grants-public-records.html':
#             BODYFIELD_SPAN_XPATH,
#         'justice-and-prev/grants/': CATEGORY_H2_XPATH,
#         'justice-and-prev/grants/bgp/': CATEGORY_H2_XPATH,
#         'hwy-safety/grants/': CATEGORY_H2_XPATH,
#         'hwy-safety/grants/ffy-2017-traffic-enforcement-grant-program.html':
#             BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/ffy2017-hsd-grant-opportunities.html':
#             BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/ffy-2017-step.html': BODYFIELD_SPAN_XPATH,
#         'hwy-safety/grants/highway-safety-grants-public-records.html':
#             BODYFIELD_SPAN_XPATH
#     }
# 
# Path: bidwire/scrapers/massgov/results_page_scraper.py
# SITE_ROOT = 'https://www.mass.gov'
# def scrape_results_page(page_str, xpath_list):


# Path: bidwire/scrapers/massgov_eopss_scraper.py
from document import Document, get_new_urls
from .base_scraper import BaseScraper
from .massgov import url_scraper_dict
from .massgov import results_page_scraper
import logging
import scrapelib




log = logging.getLogger(__name__)

URL_PREFIX = 'http://www.mass.gov/eopss/funding-and-training/'


class MassGovEOPSSScraper(BaseScraper):
    def __init__(self):
        self.url_dict = url_scraper_dict.MASSGOV_DICT


    def get_site(self):