# Path: config/base_settings.py
# BASE_PATH = os.path.split(os.path.split(__file__)[0])[0]
# OUTPUT_PATH = os.path.join(BASE_PATH, 'output')
# OUTPUT_GROUP_PATH = os.path.join(OUTPUT_PATH, TimeUtil.getFormatTime('%Y%m%d_%H%M%S'))
# DATABASE_NAME = os.path.join(OUTPUT_GROUP_PATH, 'Patent.db')
# EXCEL_NAME = os.path.join(OUTPUT_GROUP_PATH, '专利.xlsx')
# CHARTS_NAME = os.path.join(OUTPUT_GROUP_PATH, 'charts.html')
# LOG_FILENAME = os.path.join(OUTPUT_GROUP_PATH, "PatentCrawler.log")
# CAPTCHA_MODEL_NAME = os.path.join(BASE_PATH, 'res', 'captcha', 'sipo3.job')
# AD_PATH = os.path.join(BASE_PATH, 'res', 'advertisement', 'ad.html')
# USE_PROXY = False
# PROXY_URL = 'http://127.0.0.1:5010/get'
# TIMEOUT = 10
# DOWNLOAD_DELAY = 1
# OUTPUT_ITEMS = ['data', 'log', 'chart']
#         USE_PROXY = use_proxy
#                 PROXY_URL = proxy_url
#             TIMEOUT = timeout
#             DOWNLOAD_DELAY = delay
#             OUTPUT_ITEMS = output_items
# def check_proxy(cfg):
# def check_request(cfg: configparser.ConfigParser):
# def check_output(cfg):
# 
# Path: entity/models.py
# class Patents(BaseModel):
# 
#     # 行号
#     row_id = PrimaryKeyField()
#     # 专利id
#     patent_id = CharField(unique=True)
#     # 申请号
#     request_number = CharField()
#     # 申请日
#     request_date = DateField()
#     # 公开（公告）号
#     publish_number = CharField()
#     # 公开（公告）日
#     publish_date = DateField()
#     # 发明名称
#     invention_name = CharField()
#     # 申请（专利权）人
#     proposer = CharField()
#     # 发明人
#     inventor = CharField()
#     # 法律状态
#     legal_status = CharField(null=True)
#     #  法律状态生效日期
#     legal_status_effective_date = DateField(null=True)
#     # 摘要
#     abstract = TextField(null=True)
#     # IPC分类号
#     ipc_class_number = CharField(null=True)
#     # 优先权号
#     priority_number = CharField(null=True)
#     # 优先权日
#     priority_date = DateField(null=True)
#     # 外观设计洛迦诺分类号
#     locarno_class_number = CharField(null=True)
#     # 代理人
#     agent = CharField(null=True)
#     # 代理机构
#     agency = CharField(null=True)
#     # 申请人邮编
#     proposer_post_code = CharField(null=True)
#     # 申请人地址
#     proposer_address = CharField(null=True)
#     # 申请人所在国（省）
#     proposer_location = CharField(null=True)
#     # 发明类型
#     invention_type = CharField(null=True)
#     # 公开国
#     publish_country = CharField(null=True)
#     # 权利要求
#     claim = CharField(null=True)
#     # 说明书
#     instructions = TextField(null=True)
#     # FT分类号
#     FT_class_number = CharField(null=True)
#     # UC分类号
#     UC_class_number = CharField(null=True)
#     # ECLA分类号
#     ECLA_class_number = CharField(null=True)
#     # FI分类号
#     FI_class_number = CharField(null=True)
#     # CPC分类号
#     CPC_class_number = CharField(null=True)
# 
# Path: service/info.py
# def init_crawler(cfg: configparser.ConfigParser):
# def gen_crawler_list(content_list):
# def push_crawler_list():
# def create_tables():


import copy
import sqlite3
from logbook import Logger
from config import base_settings as bs
from entity.models import Patents
from service import info
# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html



logger = Logger(__name__)


class CrawlerPipeline(object):
    LINE_INDEX = 1

    def process_item(self, item, spider):
        data = item.get('data').__dict__
        table_dict = {}
        logger.info('采集内容：%s' % str(data))

# Path: config/base_settings.py
# BASE_PATH = os.path.split(os.path.split(__file__)[0])[0]
# OUTPUT_PATH = os.path.join(BASE_PATH, 'output')
# OUTPUT_GROUP_PATH = os.path.join(OUTPUT_PATH, TimeUtil.getFormatTime('%Y%m%d_%H%M%S'))
# DATABASE_NAME = os.path.join(OUTPUT_GROUP_PATH, 'Patent.db')
# EXCEL_NAME = os.path.join(OUTPUT_GROUP_PATH, '专利.xlsx')
# CHARTS_NAME = os.path.join(OUTPUT_GROUP_PATH, 'charts.html')
# LOG_FILENAME = os.path.join(OUTPUT_GROUP_PATH, "PatentCrawler.log")
# CAPTCHA_MODEL_NAME = os.path.join(BASE_PATH, 'res', 'captcha', 'sipo3.job')
# AD_PATH = os.path.join(BASE_PATH, 'res', 'advertisement', 'ad.html')
# USE_PROXY = False
# PROXY_URL = 'http://127.0.0.1:5010/get'
# TIMEOUT = 10
# DOWNLOAD_DELAY = 1
# OUTPUT_ITEMS = ['data', 'log', 'chart']
#         USE_PROXY = use_proxy
#                 PROXY_URL = proxy_url
#             TIMEOUT = timeout
#             DOWNLOAD_DELAY = delay
#             OUTPUT_ITEMS = output_items
# def check_proxy(cfg):
# def check_request(cfg: configparser.ConfigParser):
# def check_output(cfg):
# 
# Path: entity/models.py
# class Patents(BaseModel):
# 
#     # 行号
#     row_id = PrimaryKeyField()
#     # 专利id
#     patent_id = CharField(unique=True)
#     # 申请号
#     request_number = CharField()
#     # 申请日
#     request_date = DateField()
#     # 公开（公告）号
#     publish_number = CharField()
#     # 公开（公告）日
#     publish_date = DateField()
#     # 发明名称
#     invention_name = CharField()
#     # 申请（专利权）人
#     proposer = CharField()
#     # 发明人
#     inventor = CharField()
#     # 法律状态
#     legal_status = CharField(null=True)
#     #  法律状态生效日期
#     legal_status_effective_date = DateField(null=True)
#     # 摘要
#     abstract = TextField(null=True)
#     # IPC分类号
#     ipc_class_number = CharField(null=True)
#     # 优先权号
#     priority_number = CharField(null=True)
#     # 优先权日
#     priority_date = DateField(null=True)
#     # 外观设计洛迦诺分类号
#     locarno_class_number = CharField(null=True)
#     # 代理人
#     agent = CharField(null=True)
#     # 代理机构
#     agency = CharField(null=True)
#     # 申请人邮编
#     proposer_post_code = CharField(null=True)
#     # 申请人地址
#     proposer_address = CharField(null=True)
#     # 申请人所在国（省）
#     proposer_location = CharField(null=True)
#     # 发明类型
#     invention_type = CharField(null=True)
#     # 公开国
#     publish_country = CharField(null=True)
#     # 权利要求
#     claim = CharField(null=True)
#     # 说明书
#     instructions = TextField(null=True)
#     # FT分类号
#     FT_class_number = CharField(null=True)
#     # UC分类号
#     UC_class_number = CharField(null=True)
#     # ECLA分类号
#     ECLA_class_number = CharField(null=True)
#     # FI分类号
#     FI_class_number = CharField(null=True)
#     # CPC分类号
#     CPC_class_number = CharField(null=True)
# 
# Path: service/info.py
# def init_crawler(cfg: configparser.ConfigParser):
# def gen_crawler_list(content_list):
# def push_crawler_list():
# def create_tables():


# Path: crawler/pipelines.py
import copy
import sqlite3
from logbook import Logger
from config import base_settings as bs
from entity.models import Patents
from service import info

# -*- coding: utf-8 -*-

# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html



logger = Logger(__name__)


class CrawlerPipeline(object):
    LINE_INDEX = 1

    def process_item(self, item, spider):
        data = item.get('data').__dict__
        table_dict = {}
        logger.info('采集内容：%s' % str(data))

        for name, fields in info.data_table.items():