# Path: crawler/crawling/spiders/lxmlhtml.py
# class CustomLxmlLinkExtractor(LxmlLinkExtractor):
#     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(),
#                  restrict_xpaths=(),
#                  tags=('a', 'area'), attrs=('href',), canonicalize=True,
#                  unique=True, process_value=None, deny_extensions=None,
#                  restrict_css=()):
#         super(CustomLxmlLinkExtractor, self).__init__(allow=allow, deny=deny,
#                 allow_domains=allow_domains, deny_domains=deny_domains,
#                 restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
#                 canonicalize=canonicalize, deny_extensions=deny_extensions)
#         tag_func = lambda x: x in tags
#         attr_func = lambda x: x in attrs
# 
#         # custom parser override
#         cp = CustomParser(tag=tag_func, attr=attr_func,
#                           unique=unique, process=process_value)
#         self.link_extractor = cp
# 
# Path: crawler/crawling/spiders/redis_spider.py
# class RedisSpider(Spider):
#     '''
#     Base Spider for doing distributed crawls coordinated through Redis
#     '''
# 
#     def _set_crawler(self, crawler):
#         super(RedisSpider, self)._set_crawler(crawler)
#         self.crawler.signals.connect(self.spider_idle,
#                                      signal=signals.spider_idle)
# 
#     def spider_idle(self):
#         raise DontCloseSpider
# 
#     def parse(self, response):
#         '''
#         Parse a page of html, and yield items into the item pipeline
# 
#         @param response: The response object of the scrape
#         '''
#         raise NotImplementedError("Please implement parse() for your spider")
# 
#     def set_logger(self, logger):
#         '''
#         Set the logger for the spider, different than the default Scrapy one
# 
#         @param logger: the logger from the scheduler
#         '''
#         self._logger = logger
# 
#     def reconstruct_headers(self, response):
#         """
#         Purpose of this method is to reconstruct the headers dictionary that
#         is normally passed in with a "response" object from scrapy.
# 
#         Args:
#             response: A scrapy response object
# 
#         Returns: A dictionary that mirrors the "response.headers" dictionary
#         that is normally within a response object
# 
#         Raises: None
#         Reason: Originally, there was a bug where the json.dumps() did not
#         properly serialize the headers. This method is a way to circumvent
#         the known issue
#         """
# 
#         header_dict = {}
#         # begin reconstructing headers from scratch...
#         for key in list(response.headers.keys()):
#             key_item_list = []
#             key_list = response.headers.getlist(key)
#             for item in key_list:
#                 key_item_list.append(item)
#             header_dict[key] = key_item_list
#         return header_dict


import scrapy
import random
from scrapy.http import Request
from .lxmlhtml import CustomLxmlLinkExtractor as LinkExtractor
from scrapy.conf import settings
from crawling.items import RawResponseItem
from .redis_spider import RedisSpider
        # step counter for how many pages we have hit
        step = 0
        if 'step' in response.meta:
            step = response.meta['step']

        # Create Item to send to kafka
        # capture raw response
        item = RawResponseItem()
        # populated from response.meta
        item['appid'] = response.meta['appid']
        item['crawlid'] = response.meta['crawlid']
        item['attrs'] = response.meta['attrs']
        # populated from raw HTTP response
        item["url"] = response.request.url
        item["response_url"] = response.url
        item["status_code"] = response.status
        item["status_msg"] = "OK"
        item["response_headers"] = self.reconstruct_headers(response)
        item["request_headers"] = response.request.headers
        item["body"] = response.body
        item["links"] = []
        # we want to know how far our spider gets
        if item['attrs'] is None:
            item['attrs'] = {}

        item['attrs']['step'] = step

        self._logger.debug("Finished creating item")

        # determine what link we want to crawl

# Path: crawler/crawling/spiders/lxmlhtml.py
# class CustomLxmlLinkExtractor(LxmlLinkExtractor):
#     def __init__(self, allow=(), deny=(), allow_domains=(), deny_domains=(),
#                  restrict_xpaths=(),
#                  tags=('a', 'area'), attrs=('href',), canonicalize=True,
#                  unique=True, process_value=None, deny_extensions=None,
#                  restrict_css=()):
#         super(CustomLxmlLinkExtractor, self).__init__(allow=allow, deny=deny,
#                 allow_domains=allow_domains, deny_domains=deny_domains,
#                 restrict_xpaths=restrict_xpaths, restrict_css=restrict_css,
#                 canonicalize=canonicalize, deny_extensions=deny_extensions)
#         tag_func = lambda x: x in tags
#         attr_func = lambda x: x in attrs
# 
#         # custom parser override
#         cp = CustomParser(tag=tag_func, attr=attr_func,
#                           unique=unique, process=process_value)
#         self.link_extractor = cp
# 
# Path: crawler/crawling/spiders/redis_spider.py
# class RedisSpider(Spider):
#     '''
#     Base Spider for doing distributed crawls coordinated through Redis
#     '''
# 
#     def _set_crawler(self, crawler):
#         super(RedisSpider, self)._set_crawler(crawler)
#         self.crawler.signals.connect(self.spider_idle,
#                                      signal=signals.spider_idle)
# 
#     def spider_idle(self):
#         raise DontCloseSpider
# 
#     def parse(self, response):
#         '''
#         Parse a page of html, and yield items into the item pipeline
# 
#         @param response: The response object of the scrape
#         '''
#         raise NotImplementedError("Please implement parse() for your spider")
# 
#     def set_logger(self, logger):
#         '''
#         Set the logger for the spider, different than the default Scrapy one
# 
#         @param logger: the logger from the scheduler
#         '''
#         self._logger = logger
# 
#     def reconstruct_headers(self, response):
#         """
#         Purpose of this method is to reconstruct the headers dictionary that
#         is normally passed in with a "response" object from scrapy.
# 
#         Args:
#             response: A scrapy response object
# 
#         Returns: A dictionary that mirrors the "response.headers" dictionary
#         that is normally within a response object
# 
#         Raises: None
#         Reason: Originally, there was a bug where the json.dumps() did not
#         properly serialize the headers. This method is a way to circumvent
#         the known issue
#         """
# 
#         header_dict = {}
#         # begin reconstructing headers from scratch...
#         for key in list(response.headers.keys()):
#             key_item_list = []
#             key_list = response.headers.getlist(key)
#             for item in key_list:
#                 key_item_list.append(item)
#             header_dict[key] = key_item_list
#         return header_dict


# Path: crawler/crawling/spiders/wandering_spider.py
import scrapy
import random
from scrapy.http import Request
from .lxmlhtml import CustomLxmlLinkExtractor as LinkExtractor
from scrapy.conf import settings
from crawling.items import RawResponseItem
from .redis_spider import RedisSpider

        # step counter for how many pages we have hit
        step = 0
        if 'step' in response.meta:
            step = response.meta['step']

        # Create Item to send to kafka
        # capture raw response
        item = RawResponseItem()
        # populated from response.meta
        item['appid'] = response.meta['appid']
        item['crawlid'] = response.meta['crawlid']
        item['attrs'] = response.meta['attrs']
        # populated from raw HTTP response
        item["url"] = response.request.url
        item["response_url"] = response.url
        item["status_code"] = response.status
        item["status_msg"] = "OK"
        item["response_headers"] = self.reconstruct_headers(response)
        item["request_headers"] = response.request.headers
        item["body"] = response.body
        item["links"] = []
        # we want to know how far our spider gets
        if item['attrs'] is None:
            item['attrs'] = {}

        item['attrs']['step'] = step

        self._logger.debug("Finished creating item")

        # determine what link we want to crawl

        link_extractor = LinkExtractor(