# Path: firmware/items.py
# class FirmwareImage(Item):
#     category = Field(default=None)
#     vendor = Field()
#     product = Field(default=None)
# 
#     description = Field(default=None)
#     version = Field(default=None)
#     build = Field(default=None)
#     date = Field(default=None)
#     size = Field(default=None)
#     language = Field(default=None)
#     
#     mib = Field(default=None)
#     sdk = Field(default=None)
#     url = Field()
# 
#     # used by FilesPipeline
#     files = Field()
#     file_urls = Field()
# 
# Path: firmware/loader.py
# class FirmwareLoader(ItemLoader):
# 
#     @staticmethod
#     def find_product(text):
#         match = re.search(r"(?:model[:. #]*([\w-][\w.-]+))", " ".join(
#             text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version(text):
#         match = re.search(r"(?:version[:. ]*([\w-][\w.-]+)|ve?r?s?i?o?n?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_build(text):
#         match = re.search(r"(?:build[:. ]*([\w-][\w.-]+)|bu?i?l?d?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version_period(text):
#         match = re.search(r"((?:[0-9])(?:[\w-]*\.[\w-]*)+)",
#                           " ".join(text).replace("\xa0", " ").strip())
#         return next((x for x in match.groups() if x and "192.168." not in x.lower()), None) if match else None
# 
#     def find_date(self, text):
#         for fmt in self.context.get("date_fmt", []):
#             fmt = "(" + re.escape(fmt).replace("%b", "[a-zA-Z]{3}").replace("%B", "[a-zA-Z]+").replace(
#                  "%m", "\d{1,2}").replace("%d", "\d{1,2}").replace("%y", "\d{2}").replace("%Y", "\d{4}") + ")"
#             match = re.search(fmt, "".join(text).strip())
#             res = [x for x in match.groups() if x] if match else None
# 
#             if res:
#                 return res[0]
#         return None
# 
#     def clean(s):
#         return "".join([x for x in s if x in string.printable]).replace("\r", "").replace("\n", "").replace("\xa0", " ").strip()
# 
#     def fix_url(url, loader_context):
#         if not urllib.parse.urlparse(url).netloc:
#             return urllib.parse.urljoin(loader_context.get("response").url, url)
#         return url
# 
#     def parse_date(date, loader_context):
#         for fmt in loader_context.get("date_fmt", []):
#             try:
#                 return datetime.datetime.strptime(date, fmt)
#             except ValueError:
#                 pass
#         return None
# 
#     def remove_html(s):
#         return re.sub(r"<[a-zA-Z0-9\"/=: ]+>", "", s)
# 
#     default_output_processor = TakeFirst()
# 
#     product_in = MapCompose(clean)
#     vendor_in = Identity()
# 
#     description_in = MapCompose(remove_html, clean)
#     version_in = MapCompose(clean)
#     build_in = MapCompose(clean)
#     date_in = MapCompose(clean, parse_date)
# 
#     mib_in = MapCompose(fix_url)
#     sdk_in = MapCompose(fix_url)
#     url_in = MapCompose(fix_url)


from scrapy import Spider
from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader
                ("602W", "3.7.25"),
                ("602W", "5.1.1.1.5"),
                ("MINI319", "0.22.2.34"),
                ("MINI319", "0.37.2.36"),
                ("MINI319", "32.37.2.39"),
                ("MINI319", "2.7.25"),
                ("MINI319", "2.7.25"),
                ("MINI319", "2.7.25"),
                ("MINI319", "12.1.1.1.2"),
                ("ROBOT3", "1.3.3.3"),
                ("ROBOT3", "1.2.7.2"),
                ("ROBOT3", "1.2.1.4"),
                ("ROBOT3", "7.1.1.1.1.2"),
                ("ROBOT3", None),
                ("391W HD", "1.3.3.3"),
                ("391W HD", "1.2.7.2"),
                ("TH671", "8.1.1.1.1.2"),
                ("TH692", None),
                ("TH661", "7.1.1.1.1.2"),
                ("TH661", None),
                ("JPT3815W HD", None),
                ("JPT3815W HD", None)]

    def parse(self, response):
        for entry in response.xpath("//div[@class='content']//a"):
            text = entry.xpath(".//text()").extract()
            href = entry.xpath("./@href").extract()[0]

            idx = None
            for string in text:

# Path: firmware/items.py
# class FirmwareImage(Item):
#     category = Field(default=None)
#     vendor = Field()
#     product = Field(default=None)
# 
#     description = Field(default=None)
#     version = Field(default=None)
#     build = Field(default=None)
#     date = Field(default=None)
#     size = Field(default=None)
#     language = Field(default=None)
#     
#     mib = Field(default=None)
#     sdk = Field(default=None)
#     url = Field()
# 
#     # used by FilesPipeline
#     files = Field()
#     file_urls = Field()
# 
# Path: firmware/loader.py
# class FirmwareLoader(ItemLoader):
# 
#     @staticmethod
#     def find_product(text):
#         match = re.search(r"(?:model[:. #]*([\w-][\w.-]+))", " ".join(
#             text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version(text):
#         match = re.search(r"(?:version[:. ]*([\w-][\w.-]+)|ve?r?s?i?o?n?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_build(text):
#         match = re.search(r"(?:build[:. ]*([\w-][\w.-]+)|bu?i?l?d?[:. ]*([\d-][\w.-]+))",
#                           " ".join(text).replace("\xa0", " ").strip(), flags=re.IGNORECASE)
#         return next((x for x in match.groups() if x), None) if match else None
# 
#     @staticmethod
#     def find_version_period(text):
#         match = re.search(r"((?:[0-9])(?:[\w-]*\.[\w-]*)+)",
#                           " ".join(text).replace("\xa0", " ").strip())
#         return next((x for x in match.groups() if x and "192.168." not in x.lower()), None) if match else None
# 
#     def find_date(self, text):
#         for fmt in self.context.get("date_fmt", []):
#             fmt = "(" + re.escape(fmt).replace("%b", "[a-zA-Z]{3}").replace("%B", "[a-zA-Z]+").replace(
#                  "%m", "\d{1,2}").replace("%d", "\d{1,2}").replace("%y", "\d{2}").replace("%Y", "\d{4}") + ")"
#             match = re.search(fmt, "".join(text).strip())
#             res = [x for x in match.groups() if x] if match else None
# 
#             if res:
#                 return res[0]
#         return None
# 
#     def clean(s):
#         return "".join([x for x in s if x in string.printable]).replace("\r", "").replace("\n", "").replace("\xa0", " ").strip()
# 
#     def fix_url(url, loader_context):
#         if not urllib.parse.urlparse(url).netloc:
#             return urllib.parse.urljoin(loader_context.get("response").url, url)
#         return url
# 
#     def parse_date(date, loader_context):
#         for fmt in loader_context.get("date_fmt", []):
#             try:
#                 return datetime.datetime.strptime(date, fmt)
#             except ValueError:
#                 pass
#         return None
# 
#     def remove_html(s):
#         return re.sub(r"<[a-zA-Z0-9\"/=: ]+>", "", s)
# 
#     default_output_processor = TakeFirst()
# 
#     product_in = MapCompose(clean)
#     vendor_in = Identity()
# 
#     description_in = MapCompose(remove_html, clean)
#     version_in = MapCompose(clean)
#     build_in = MapCompose(clean)
#     date_in = MapCompose(clean, parse_date)
# 
#     mib_in = MapCompose(fix_url)
#     sdk_in = MapCompose(fix_url)
#     url_in = MapCompose(fix_url)


# Path: firmware/spiders/tenvis.py
from scrapy import Spider
from firmware.items import FirmwareImage
from firmware.loader import FirmwareLoader

                ("602W", "3.7.25"),
                ("602W", "5.1.1.1.5"),
                ("MINI319", "0.22.2.34"),
                ("MINI319", "0.37.2.36"),
                ("MINI319", "32.37.2.39"),
                ("MINI319", "2.7.25"),
                ("MINI319", "2.7.25"),
                ("MINI319", "2.7.25"),
                ("MINI319", "12.1.1.1.2"),
                ("ROBOT3", "1.3.3.3"),
                ("ROBOT3", "1.2.7.2"),
                ("ROBOT3", "1.2.1.4"),
                ("ROBOT3", "7.1.1.1.1.2"),
                ("ROBOT3", None),
                ("391W HD", "1.3.3.3"),
                ("391W HD", "1.2.7.2"),
                ("TH671", "8.1.1.1.1.2"),
                ("TH692", None),
                ("TH661", "7.1.1.1.1.2"),
                ("TH661", None),
                ("JPT3815W HD", None),
                ("JPT3815W HD", None)]

    def parse(self, response):
        for entry in response.xpath("//div[@class='content']//a"):
            text = entry.xpath(".//text()").extract()
            href = entry.xpath("./@href").extract()[0]

            idx = None
            for string in text:

                if "---" in string: