# Path: firmware/items.py
# class FirmwareImage(Item):
#     category = Field(default=None)
#     vendor = Field()
#     product = Field(default=None)
# 
#     description = Field(default=None)
#     version = Field(default=None)
#     build = Field(default=None)
#     date = Field(default=None)
#     size = Field(default=None)
#     language = Field(default=None)
#     
#     mib = Field(default=None)
#     sdk = Field(default=None)
#     url = Field()
# 
#     # used by FilesPipeline
#     files = Field()
#     file_urls = Field()
# 
# Path: firmware/loader.py
# class FirmwareLoader(ItemLoader):
# 
#     @staticmethod
#     def find_product(text):
#         match = re.search(r"(?:model[:. #]*([\w-][\w.-]+))", " ".join(
#             text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version(text):
#         match = re.search(r"(?:version[:. ]*([\w-][\w.-]+)|ve?r?s?i?o?n?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_build(text):
#         match = re.search(r"(?:build[:. ]*([\w-][\w.-]+)|bu?i?l?d?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version_period(text):
#         match = re.search(r"((?:[0-9])(?:[\w-]*\.[\w-]*)+)",
#                           " ".join(text).replace("\xa0", " ").strip())
#         return next((x for x in match.groups() if x and "192.168." not in x.lower()), None) if match else None
# 
#     def find_date(self, text):
#         for fmt in self.context.get("date_fmt", []):
#             fmt = "(" + re.escape(fmt).replace("%b", "[a-zA-Z]{3}").replace("%B", "[a-zA-Z]+").replace(
#                  "%m", "\d{1,2}").replace("%d", "\d{1,2}").replace("%y", "\d{2}").replace("%Y", "\d{4}") + ")"
#             match = re.search(fmt, "".join(text).strip())
#             res = [x for x in match.groups() if x] if match else None
# 
#             if res:
#                 return res[0]
#         return None
# 
#     def clean(s):
#         return "".join([x for x in s if x in string.printable]).replace("\r", "").replace("\n", "").replace("\xa0", " ").strip()
# 
#     def fix_url(url, loader_context):
#         if not urllib.parse.urlparse(url).netloc:
#             return urllib.parse.urljoin(loader_context.get("response").url, url)
#         return url
# 
#     def parse_date(date, loader_context):
#         for fmt in loader_context.get("date_fmt", []):
#             try:
#                 return datetime.datetime.strptime(date, fmt)
#             except ValueError:
#                 pass
#         return None
# 
#     def remove_html(s):
#         return re.sub(r"<[a-zA-Z0-9\"/=: ]+>", "", s)
# 
#     default_output_processor = TakeFirst()
# 
#     product_in = MapCompose(clean)
#     vendor_in = Identity()
# 
#     description_in = MapCompose(remove_html, clean)
#     version_in = MapCompose(clean)
#     build_in = MapCompose(clean)
#     date_in = MapCompose(clean, parse_date)
# 
#     mib_in = MapCompose(fix_url)
#     sdk_in = MapCompose(fix_url)
#     url_in = MapCompose(fix_url)


from scrapy import Spider
from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader
import urllib.request, urllib.parse, urllib.error



class TPLinkENSpider(Spider):
    name = "tp-link_en"
    vendor = "tp-link"

    start_urls = ["https://www.tp-link.com/hk/support/download/"]

    def parse(self, response):
        for product_group in response.css("div.item"):
            try:
                category = product_group.css("h2 span::text").get().strip()
            except:
                category = product_group.css("h2::text").get().strip()
            for product in product_group.css("a"):
                model = product.css("::text").get().strip()
                link = product.css("::attr(href)").get()
                    
                if link[-1] == "/":
                    yield response.follow(link, meta={"category": category, "product":model}, callback=self.parse_product)
                else:
                    item = FirmwareLoader(item=FirmwareImage(), date_fmt=["%Y-%m-%d"])

# Path: firmware/items.py
# class FirmwareImage(Item):
#     category = Field(default=None)
#     vendor = Field()
#     product = Field(default=None)
# 
#     description = Field(default=None)
#     version = Field(default=None)
#     build = Field(default=None)
#     date = Field(default=None)
#     size = Field(default=None)
#     language = Field(default=None)
#     
#     mib = Field(default=None)
#     sdk = Field(default=None)
#     url = Field()
# 
#     # used by FilesPipeline
#     files = Field()
#     file_urls = Field()
# 
# Path: firmware/loader.py
# class FirmwareLoader(ItemLoader):
# 
#     @staticmethod
#     def find_product(text):
#         match = re.search(r"(?:model[:. #]*([\w-][\w.-]+))", " ".join(
#             text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version(text):
#         match = re.search(r"(?:version[:. ]*([\w-][\w.-]+)|ve?r?s?i?o?n?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_build(text):
#         match = re.search(r"(?:build[:. ]*([\w-][\w.-]+)|bu?i?l?d?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version_period(text):
#         match = re.search(r"((?:[0-9])(?:[\w-]*\.[\w-]*)+)",
#                           " ".join(text).replace("\xa0", " ").strip())
#         return next((x for x in match.groups() if x and "192.168." not in x.lower()), None) if match else None
# 
#     def find_date(self, text):
#         for fmt in self.context.get("date_fmt", []):
#             fmt = "(" + re.escape(fmt).replace("%b", "[a-zA-Z]{3}").replace("%B", "[a-zA-Z]+").replace(
#                  "%m", "\d{1,2}").replace("%d", "\d{1,2}").replace("%y", "\d{2}").replace("%Y", "\d{4}") + ")"
#             match = re.search(fmt, "".join(text).strip())
#             res = [x for x in match.groups() if x] if match else None
# 
#             if res:
#                 return res[0]
#         return None
# 
#     def clean(s):
#         return "".join([x for x in s if x in string.printable]).replace("\r", "").replace("\n", "").replace("\xa0", " ").strip()
# 
#     def fix_url(url, loader_context):
#         if not urllib.parse.urlparse(url).netloc:
#             return urllib.parse.urljoin(loader_context.get("response").url, url)
#         return url
# 
#     def parse_date(date, loader_context):
#         for fmt in loader_context.get("date_fmt", []):
#             try:
#                 return datetime.datetime.strptime(date, fmt)
#             except ValueError:
#                 pass
#         return None
# 
#     def remove_html(s):
#         return re.sub(r"<[a-zA-Z0-9\"/=: ]+>", "", s)
# 
#     default_output_processor = TakeFirst()
# 
#     product_in = MapCompose(clean)
#     vendor_in = Identity()
# 
#     description_in = MapCompose(remove_html, clean)
#     version_in = MapCompose(clean)
#     build_in = MapCompose(clean)
#     date_in = MapCompose(clean, parse_date)
# 
#     mib_in = MapCompose(fix_url)
#     sdk_in = MapCompose(fix_url)
#     url_in = MapCompose(fix_url)


# Path: firmware/spiders/tp-link_en.py
from scrapy import Spider
from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader
import urllib.request, urllib.parse, urllib.error




class TPLinkENSpider(Spider):
    name = "tp-link_en"
    vendor = "tp-link"

    start_urls = ["https://www.tp-link.com/hk/support/download/"]

    def parse(self, response):
        for product_group in response.css("div.item"):
            try:
                category = product_group.css("h2 span::text").get().strip()
            except:
                category = product_group.css("h2::text").get().strip()
            for product in product_group.css("a"):
                model = product.css("::text").get().strip()
                link = product.css("::attr(href)").get()
                    
                if link[-1] == "/":
                    yield response.follow(link, meta={"category": category, "product":model}, callback=self.parse_product)
                else:
                    item = FirmwareLoader(item=FirmwareImage(), date_fmt=["%Y-%m-%d"])

                    item.add_value("vendor", self.vendor)