# Path: firmware/items.py
# class FirmwareImage(Item):
#     category = Field(default=None)
#     vendor = Field()
#     product = Field(default=None)
# 
#     description = Field(default=None)
#     version = Field(default=None)
#     build = Field(default=None)
#     date = Field(default=None)
#     size = Field(default=None)
#     language = Field(default=None)
#     
#     mib = Field(default=None)
#     sdk = Field(default=None)
#     url = Field()
# 
#     # used by FilesPipeline
#     files = Field()
#     file_urls = Field()
# 
# Path: firmware/loader.py
# class FirmwareLoader(ItemLoader):
# 
#     @staticmethod
#     def find_product(text):
#         match = re.search(r"(?:model[:. #]*([\w-][\w.-]+))", " ".join(
#             text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version(text):
#         match = re.search(r"(?:version[:. ]*([\w-][\w.-]+)|ve?r?s?i?o?n?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_build(text):
#         match = re.search(r"(?:build[:. ]*([\w-][\w.-]+)|bu?i?l?d?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version_period(text):
#         match = re.search(r"((?:[0-9])(?:[\w-]*\.[\w-]*)+)",
#                           " ".join(text).replace("\xa0", " ").strip())
#         return next((x for x in match.groups() if x and "192.168." not in x.lower()), None) if match else None
# 
#     def find_date(self, text):
#         for fmt in self.context.get("date_fmt", []):
#             fmt = "(" + re.escape(fmt).replace("%b", "[a-zA-Z]{3}").replace("%B", "[a-zA-Z]+").replace(
#                  "%m", "\d{1,2}").replace("%d", "\d{1,2}").replace("%y", "\d{2}").replace("%Y", "\d{4}") + ")"
#             match = re.search(fmt, "".join(text).strip())
#             res = [x for x in match.groups() if x] if match else None
# 
#             if res:
#                 return res[0]
#         return None
# 
#     def clean(s):
#         return "".join([x for x in s if x in string.printable]).replace("\r", "").replace("\n", "").replace("\xa0", " ").strip()
# 
#     def fix_url(url, loader_context):
#         if not urllib.parse.urlparse(url).netloc:
#             return urllib.parse.urljoin(loader_context.get("response").url, url)
#         return url
# 
#     def parse_date(date, loader_context):
#         for fmt in loader_context.get("date_fmt", []):
#             try:
#                 return datetime.datetime.strptime(date, fmt)
#             except ValueError:
#                 pass
#         return None
# 
#     def remove_html(s):
#         return re.sub(r"<[a-zA-Z0-9\"/=: ]+>", "", s)
# 
#     default_output_processor = TakeFirst()
# 
#     product_in = MapCompose(clean)
#     vendor_in = Identity()
# 
#     description_in = MapCompose(remove_html, clean)
#     version_in = MapCompose(clean)
#     build_in = MapCompose(clean)
#     date_in = MapCompose(clean, parse_date)
# 
#     mib_in = MapCompose(fix_url)
#     sdk_in = MapCompose(fix_url)
#     url_in = MapCompose(fix_url)


from scrapy import Spider
from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader
import urllib.request, urllib.parse, urllib.error
        version = ""
        if tmp[0] != 'v':
            links=response.css("div.hardware-version dl.select-version li a::attr(href)").extract()
            if len(links):
                version = links[0].split('/')[-2]
                del links[0]
                for link in links:
                    yield response.follow(link, meta=response.meta, callback=self.parse_product)
        
        firmwares=response.css("#content_Firmware > table")
        self.logger.debug("%s %s: %d binary firmware found." % (response.meta["product"], version, len(firmwares)))
        for firmware in firmwares:
            spans = firmware.css('tr.detail-info span')
            item = FirmwareLoader(
            item=FirmwareImage(), response=response, date_fmt=["%Y-%m-%d"])
            item.add_value("vendor", self.vendor)
            item.add_value("url", firmware.css("a::attr(href)").get())
            item.add_value("date", spans[1].css("::text").get().strip())
            item.add_value("language", spans[3].css("::text").get().strip())
            item.add_value("size", spans[5].css("::text").get().strip())
            item.add_value("description", "\n".join(firmware.css('td.more p').getall()))
            item.add_value("product", response.meta["product"])
            item.add_value("category", response.meta["category"])
            item.add_value("version", version)
            yield item.load_item()

        gpl_source_codes=response.css("#content_GPL-Code a")
        self.logger.debug("%s %s: %d gpl source code found." % (response.meta["product"], version, len(gpl_source_codes)))
        for gpl in gpl_source_codes:
            item = FirmwareLoader(

# Path: firmware/items.py
# class FirmwareImage(Item):
#     category = Field(default=None)
#     vendor = Field()
#     product = Field(default=None)
# 
#     description = Field(default=None)
#     version = Field(default=None)
#     build = Field(default=None)
#     date = Field(default=None)
#     size = Field(default=None)
#     language = Field(default=None)
#     
#     mib = Field(default=None)
#     sdk = Field(default=None)
#     url = Field()
# 
#     # used by FilesPipeline
#     files = Field()
#     file_urls = Field()
# 
# Path: firmware/loader.py
# class FirmwareLoader(ItemLoader):
# 
#     @staticmethod
#     def find_product(text):
#         match = re.search(r"(?:model[:. #]*([\w-][\w.-]+))", " ".join(
#             text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version(text):
#         match = re.search(r"(?:version[:. ]*([\w-][\w.-]+)|ve?r?s?i?o?n?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_build(text):
#         match = re.search(r"(?:build[:. ]*([\w-][\w.-]+)|bu?i?l?d?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version_period(text):
#         match = re.search(r"((?:[0-9])(?:[\w-]*\.[\w-]*)+)",
#                           " ".join(text).replace("\xa0", " ").strip())
#         return next((x for x in match.groups() if x and "192.168." not in x.lower()), None) if match else None
# 
#     def find_date(self, text):
#         for fmt in self.context.get("date_fmt", []):
#             fmt = "(" + re.escape(fmt).replace("%b", "[a-zA-Z]{3}").replace("%B", "[a-zA-Z]+").replace(
#                  "%m", "\d{1,2}").replace("%d", "\d{1,2}").replace("%y", "\d{2}").replace("%Y", "\d{4}") + ")"
#             match = re.search(fmt, "".join(text).strip())
#             res = [x for x in match.groups() if x] if match else None
# 
#             if res:
#                 return res[0]
#         return None
# 
#     def clean(s):
#         return "".join([x for x in s if x in string.printable]).replace("\r", "").replace("\n", "").replace("\xa0", " ").strip()
# 
#     def fix_url(url, loader_context):
#         if not urllib.parse.urlparse(url).netloc:
#             return urllib.parse.urljoin(loader_context.get("response").url, url)
#         return url
# 
#     def parse_date(date, loader_context):
#         for fmt in loader_context.get("date_fmt", []):
#             try:
#                 return datetime.datetime.strptime(date, fmt)
#             except ValueError:
#                 pass
#         return None
# 
#     def remove_html(s):
#         return re.sub(r"<[a-zA-Z0-9\"/=: ]+>", "", s)
# 
#     default_output_processor = TakeFirst()
# 
#     product_in = MapCompose(clean)
#     vendor_in = Identity()
# 
#     description_in = MapCompose(remove_html, clean)
#     version_in = MapCompose(clean)
#     build_in = MapCompose(clean)
#     date_in = MapCompose(clean, parse_date)
# 
#     mib_in = MapCompose(fix_url)
#     sdk_in = MapCompose(fix_url)
#     url_in = MapCompose(fix_url)


# Path: firmware/spiders/tp-link_en.py
from scrapy import Spider
from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader
import urllib.request, urllib.parse, urllib.error

        version = ""
        if tmp[0] != 'v':
            links=response.css("div.hardware-version dl.select-version li a::attr(href)").extract()
            if len(links):
                version = links[0].split('/')[-2]
                del links[0]
                for link in links:
                    yield response.follow(link, meta=response.meta, callback=self.parse_product)
        
        firmwares=response.css("#content_Firmware > table")
        self.logger.debug("%s %s: %d binary firmware found." % (response.meta["product"], version, len(firmwares)))
        for firmware in firmwares:
            spans = firmware.css('tr.detail-info span')
            item = FirmwareLoader(
            item=FirmwareImage(), response=response, date_fmt=["%Y-%m-%d"])
            item.add_value("vendor", self.vendor)
            item.add_value("url", firmware.css("a::attr(href)").get())
            item.add_value("date", spans[1].css("::text").get().strip())
            item.add_value("language", spans[3].css("::text").get().strip())
            item.add_value("size", spans[5].css("::text").get().strip())
            item.add_value("description", "\n".join(firmware.css('td.more p').getall()))
            item.add_value("product", response.meta["product"])
            item.add_value("category", response.meta["category"])
            item.add_value("version", version)
            yield item.load_item()

        gpl_source_codes=response.css("#content_GPL-Code a")
        self.logger.debug("%s %s: %d gpl source code found." % (response.meta["product"], version, len(gpl_source_codes)))
        for gpl in gpl_source_codes:
            item = FirmwareLoader(

            item=FirmwareImage(), response=response, date_fmt=["%d/%m/%Y"])