# Path: firmware/items.py
# class FirmwareImage(Item):
#     category = Field(default=None)
#     vendor = Field()
#     product = Field(default=None)
# 
#     description = Field(default=None)
#     version = Field(default=None)
#     build = Field(default=None)
#     date = Field(default=None)
#     size = Field(default=None)
#     language = Field(default=None)
#     
#     mib = Field(default=None)
#     sdk = Field(default=None)
#     url = Field()
# 
#     # used by FilesPipeline
#     files = Field()
#     file_urls = Field()
# 
# Path: firmware/loader.py
# class FirmwareLoader(ItemLoader):
# 
#     @staticmethod
#     def find_product(text):
#         match = re.search(r"(?:model[:. #]*([\w-][\w.-]+))", " ".join(
#             text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version(text):
#         match = re.search(r"(?:version[:. ]*([\w-][\w.-]+)|ve?r?s?i?o?n?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_build(text):
#         match = re.search(r"(?:build[:. ]*([\w-][\w.-]+)|bu?i?l?d?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version_period(text):
#         match = re.search(r"((?:[0-9])(?:[\w-]*\.[\w-]*)+)",
#                           " ".join(text).replace("\xa0", " ").strip())
#         return next((x for x in match.groups() if x and "192.168." not in x.lower()), None) if match else None
# 
#     def find_date(self, text):
#         for fmt in self.context.get("date_fmt", []):
#             fmt = "(" + re.escape(fmt).replace("%b", "[a-zA-Z]{3}").replace("%B", "[a-zA-Z]+").replace(
#                  "%m", "\d{1,2}").replace("%d", "\d{1,2}").replace("%y", "\d{2}").replace("%Y", "\d{4}") + ")"
#             match = re.search(fmt, "".join(text).strip())
#             res = [x for x in match.groups() if x] if match else None
# 
#             if res:
#                 return res[0]
#         return None
# 
#     def clean(s):
#         return "".join([x for x in s if x in string.printable]).replace("\r", "").replace("\n", "").replace("\xa0", " ").strip()
# 
#     def fix_url(url, loader_context):
#         if not urllib.parse.urlparse(url).netloc:
#             return urllib.parse.urljoin(loader_context.get("response").url, url)
#         return url
# 
#     def parse_date(date, loader_context):
#         for fmt in loader_context.get("date_fmt", []):
#             try:
#                 return datetime.datetime.strptime(date, fmt)
#             except ValueError:
#                 pass
#         return None
# 
#     def remove_html(s):
#         return re.sub(r"<[a-zA-Z0-9\"/=: ]+>", "", s)
# 
#     default_output_processor = TakeFirst()
# 
#     product_in = MapCompose(clean)
#     vendor_in = Identity()
# 
#     description_in = MapCompose(remove_html, clean)
#     version_in = MapCompose(clean)
#     build_in = MapCompose(clean)
#     date_in = MapCompose(clean, parse_date)
# 
#     mib_in = MapCompose(fix_url)
#     sdk_in = MapCompose(fix_url)
#     url_in = MapCompose(fix_url)


from scrapy import Spider
from scrapy.http import Request
from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader
import logging
import urllib.request, urllib.parse, urllib.error
            href = link.xpath('@href').extract_first()

            yield Request(
                url=urllib.parse.urljoin(response.url, href),
                headers={"Referer": response.url},
                meta={'product': product},
                callback=self.parse_product)

    def parse_product(self, response):
        # Find the "Software and Firmware" tab link to get to the product-range-download page
        meta = response.meta
        meta['dont_redirect'] = True
        for link in response.css('a.tab-link'):
            href = link.xpath('@href').extract_first()
            if href.endswith('software-firmware-tab'):
                logging.debug("Requesting SW+FW page for %s at %s",
                        response.meta['product'], urllib.urljoin(response.url, href))

                yield Request(
                    url=urllib.parse.urljoin(response.url, href),
                    headers={"Referer": response.url},
                    meta=meta,
                    callback=self.parse_product_sw_fw)

                break
        else:
            logging.debug("Did not find a 'Software and Firmware' tab for %s",
                    response.meta['product'])

    def parse_product_sw_fw(self, response):

# Path: firmware/items.py
# class FirmwareImage(Item):
#     category = Field(default=None)
#     vendor = Field()
#     product = Field(default=None)
# 
#     description = Field(default=None)
#     version = Field(default=None)
#     build = Field(default=None)
#     date = Field(default=None)
#     size = Field(default=None)
#     language = Field(default=None)
#     
#     mib = Field(default=None)
#     sdk = Field(default=None)
#     url = Field()
# 
#     # used by FilesPipeline
#     files = Field()
#     file_urls = Field()
# 
# Path: firmware/loader.py
# class FirmwareLoader(ItemLoader):
# 
#     @staticmethod
#     def find_product(text):
#         match = re.search(r"(?:model[:. #]*([\w-][\w.-]+))", " ".join(
#             text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version(text):
#         match = re.search(r"(?:version[:. ]*([\w-][\w.-]+)|ve?r?s?i?o?n?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_build(text):
#         match = re.search(r"(?:build[:. ]*([\w-][\w.-]+)|bu?i?l?d?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version_period(text):
#         match = re.search(r"((?:[0-9])(?:[\w-]*\.[\w-]*)+)",
#                           " ".join(text).replace("\xa0", " ").strip())
#         return next((x for x in match.groups() if x and "192.168." not in x.lower()), None) if match else None
# 
#     def find_date(self, text):
#         for fmt in self.context.get("date_fmt", []):
#             fmt = "(" + re.escape(fmt).replace("%b", "[a-zA-Z]{3}").replace("%B", "[a-zA-Z]+").replace(
#                  "%m", "\d{1,2}").replace("%d", "\d{1,2}").replace("%y", "\d{2}").replace("%Y", "\d{4}") + ")"
#             match = re.search(fmt, "".join(text).strip())
#             res = [x for x in match.groups() if x] if match else None
# 
#             if res:
#                 return res[0]
#         return None
# 
#     def clean(s):
#         return "".join([x for x in s if x in string.printable]).replace("\r", "").replace("\n", "").replace("\xa0", " ").strip()
# 
#     def fix_url(url, loader_context):
#         if not urllib.parse.urlparse(url).netloc:
#             return urllib.parse.urljoin(loader_context.get("response").url, url)
#         return url
# 
#     def parse_date(date, loader_context):
#         for fmt in loader_context.get("date_fmt", []):
#             try:
#                 return datetime.datetime.strptime(date, fmt)
#             except ValueError:
#                 pass
#         return None
# 
#     def remove_html(s):
#         return re.sub(r"<[a-zA-Z0-9\"/=: ]+>", "", s)
# 
#     default_output_processor = TakeFirst()
# 
#     product_in = MapCompose(clean)
#     vendor_in = Identity()
# 
#     description_in = MapCompose(remove_html, clean)
#     version_in = MapCompose(clean)
#     build_in = MapCompose(clean)
#     date_in = MapCompose(clean, parse_date)
# 
#     mib_in = MapCompose(fix_url)
#     sdk_in = MapCompose(fix_url)
#     url_in = MapCompose(fix_url)


# Path: firmware/spiders/se.py
from scrapy import Spider
from scrapy.http import Request
from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader
import logging
import urllib.request, urllib.parse, urllib.error

            href = link.xpath('@href').extract_first()

            yield Request(
                url=urllib.parse.urljoin(response.url, href),
                headers={"Referer": response.url},
                meta={'product': product},
                callback=self.parse_product)

    def parse_product(self, response):
        # Find the "Software and Firmware" tab link to get to the product-range-download page
        meta = response.meta
        meta['dont_redirect'] = True
        for link in response.css('a.tab-link'):
            href = link.xpath('@href').extract_first()
            if href.endswith('software-firmware-tab'):
                logging.debug("Requesting SW+FW page for %s at %s",
                        response.meta['product'], urllib.urljoin(response.url, href))

                yield Request(
                    url=urllib.parse.urljoin(response.url, href),
                    headers={"Referer": response.url},
                    meta=meta,
                    callback=self.parse_product_sw_fw)

                break
        else:
            logging.debug("Did not find a 'Software and Firmware' tab for %s",
                    response.meta['product'])

    def parse_product_sw_fw(self, response):

        product = response.meta['product']