# Path: firmware/items.py
# class FirmwareImage(Item):
#     category = Field(default=None)
#     vendor = Field()
#     product = Field(default=None)
# 
#     description = Field(default=None)
#     version = Field(default=None)
#     build = Field(default=None)
#     date = Field(default=None)
#     size = Field(default=None)
#     language = Field(default=None)
#     
#     mib = Field(default=None)
#     sdk = Field(default=None)
#     url = Field()
# 
#     # used by FilesPipeline
#     files = Field()
#     file_urls = Field()
# 
# Path: firmware/loader.py
# class FirmwareLoader(ItemLoader):
# 
#     @staticmethod
#     def find_product(text):
#         match = re.search(r"(?:model[:. #]*([\w-][\w.-]+))", " ".join(
#             text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version(text):
#         match = re.search(r"(?:version[:. ]*([\w-][\w.-]+)|ve?r?s?i?o?n?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_build(text):
#         match = re.search(r"(?:build[:. ]*([\w-][\w.-]+)|bu?i?l?d?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version_period(text):
#         match = re.search(r"((?:[0-9])(?:[\w-]*\.[\w-]*)+)",
#                           " ".join(text).replace("\xa0", " ").strip())
#         return next((x for x in match.groups() if x and "192.168." not in x.lower()), None) if match else None
# 
#     def find_date(self, text):
#         for fmt in self.context.get("date_fmt", []):
#             fmt = "(" + re.escape(fmt).replace("%b", "[a-zA-Z]{3}").replace("%B", "[a-zA-Z]+").replace(
#                  "%m", "\d{1,2}").replace("%d", "\d{1,2}").replace("%y", "\d{2}").replace("%Y", "\d{4}") + ")"
#             match = re.search(fmt, "".join(text).strip())
#             res = [x for x in match.groups() if x] if match else None
# 
#             if res:
#                 return res[0]
#         return None
# 
#     def clean(s):
#         return "".join([x for x in s if x in string.printable]).replace("\r", "").replace("\n", "").replace("\xa0", " ").strip()
# 
#     def fix_url(url, loader_context):
#         if not urllib.parse.urlparse(url).netloc:
#             return urllib.parse.urljoin(loader_context.get("response").url, url)
#         return url
# 
#     def parse_date(date, loader_context):
#         for fmt in loader_context.get("date_fmt", []):
#             try:
#                 return datetime.datetime.strptime(date, fmt)
#             except ValueError:
#                 pass
#         return None
# 
#     def remove_html(s):
#         return re.sub(r"<[a-zA-Z0-9\"/=: ]+>", "", s)
# 
#     default_output_processor = TakeFirst()
# 
#     product_in = MapCompose(clean)
#     vendor_in = Identity()
# 
#     description_in = MapCompose(remove_html, clean)
#     version_in = MapCompose(clean)
#     build_in = MapCompose(clean)
#     date_in = MapCompose(clean, parse_date)
# 
#     mib_in = MapCompose(fix_url)
#     sdk_in = MapCompose(fix_url)
#     url_in = MapCompose(fix_url)


from scrapy import Spider
from scrapy.http import FormRequest
from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader
from urllib.parse import urljoin
import json
import re
        "http://service.tp-link.com.cn/list_download_software_1_0.html"]

    def parse(self, response):
        script = response.css("script")[9].get()
        match = re.search(r'maxPage = "(\d+)"', script)
        if not match:
            return

        self.logger.debug(match.groups())
        maxPage = int(match[1])

        for page in range(1, maxPage+1):
            yield FormRequest(urljoin(response.url, '/download1/readmore'), method='GET', formdata={"ordertype": "0", "classtip": "software", "p": str(page)},
                              headers={"Referer": response.url},
                              callback=self.parse_json)

    def parse_json(self, response):
        resp = json.loads(response.text)
        self.logger.debug(resp)
        for product in resp:
            name = product['showName'].strip()
            item = FirmwareLoader(item=FirmwareImage(),
                                  response=response, date_fmt=["%Y%m%d"])

            # Model, Version, Date, Build
            self.logger.debug("Parsing '%s'" % name)
            match = re.search(
                r'^(.+) (V[\d\.]+)([^\d]+)(\d+)_([\d\.]+)$', name)

            if match:

# Path: firmware/items.py
# class FirmwareImage(Item):
#     category = Field(default=None)
#     vendor = Field()
#     product = Field(default=None)
# 
#     description = Field(default=None)
#     version = Field(default=None)
#     build = Field(default=None)
#     date = Field(default=None)
#     size = Field(default=None)
#     language = Field(default=None)
#     
#     mib = Field(default=None)
#     sdk = Field(default=None)
#     url = Field()
# 
#     # used by FilesPipeline
#     files = Field()
#     file_urls = Field()
# 
# Path: firmware/loader.py
# class FirmwareLoader(ItemLoader):
# 
#     @staticmethod
#     def find_product(text):
#         match = re.search(r"(?:model[:. #]*([\w-][\w.-]+))", " ".join(
#             text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version(text):
#         match = re.search(r"(?:version[:. ]*([\w-][\w.-]+)|ve?r?s?i?o?n?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_build(text):
#         match = re.search(r"(?:build[:. ]*([\w-][\w.-]+)|bu?i?l?d?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version_period(text):
#         match = re.search(r"((?:[0-9])(?:[\w-]*\.[\w-]*)+)",
#                           " ".join(text).replace("\xa0", " ").strip())
#         return next((x for x in match.groups() if x and "192.168." not in x.lower()), None) if match else None
# 
#     def find_date(self, text):
#         for fmt in self.context.get("date_fmt", []):
#             fmt = "(" + re.escape(fmt).replace("%b", "[a-zA-Z]{3}").replace("%B", "[a-zA-Z]+").replace(
#                  "%m", "\d{1,2}").replace("%d", "\d{1,2}").replace("%y", "\d{2}").replace("%Y", "\d{4}") + ")"
#             match = re.search(fmt, "".join(text).strip())
#             res = [x for x in match.groups() if x] if match else None
# 
#             if res:
#                 return res[0]
#         return None
# 
#     def clean(s):
#         return "".join([x for x in s if x in string.printable]).replace("\r", "").replace("\n", "").replace("\xa0", " ").strip()
# 
#     def fix_url(url, loader_context):
#         if not urllib.parse.urlparse(url).netloc:
#             return urllib.parse.urljoin(loader_context.get("response").url, url)
#         return url
# 
#     def parse_date(date, loader_context):
#         for fmt in loader_context.get("date_fmt", []):
#             try:
#                 return datetime.datetime.strptime(date, fmt)
#             except ValueError:
#                 pass
#         return None
# 
#     def remove_html(s):
#         return re.sub(r"<[a-zA-Z0-9\"/=: ]+>", "", s)
# 
#     default_output_processor = TakeFirst()
# 
#     product_in = MapCompose(clean)
#     vendor_in = Identity()
# 
#     description_in = MapCompose(remove_html, clean)
#     version_in = MapCompose(clean)
#     build_in = MapCompose(clean)
#     date_in = MapCompose(clean, parse_date)
# 
#     mib_in = MapCompose(fix_url)
#     sdk_in = MapCompose(fix_url)
#     url_in = MapCompose(fix_url)


# Path: firmware/spiders/tp-link_zh-cn.py
from scrapy import Spider
from scrapy.http import FormRequest
from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader
from urllib.parse import urljoin
import json
import re

        "http://service.tp-link.com.cn/list_download_software_1_0.html"]

    def parse(self, response):
        script = response.css("script")[9].get()
        match = re.search(r'maxPage = "(\d+)"', script)
        if not match:
            return

        self.logger.debug(match.groups())
        maxPage = int(match[1])

        for page in range(1, maxPage+1):
            yield FormRequest(urljoin(response.url, '/download1/readmore'), method='GET', formdata={"ordertype": "0", "classtip": "software", "p": str(page)},
                              headers={"Referer": response.url},
                              callback=self.parse_json)

    def parse_json(self, response):
        resp = json.loads(response.text)
        self.logger.debug(resp)
        for product in resp:
            name = product['showName'].strip()
            item = FirmwareLoader(item=FirmwareImage(),
                                  response=response, date_fmt=["%Y%m%d"])

            # Model, Version, Date, Build
            self.logger.debug("Parsing '%s'" % name)
            match = re.search(
                r'^(.+) (V[\d\.]+)([^\d]+)(\d+)_([\d\.]+)$', name)

            if match:

                self.logger.debug(match.groups())