# Path: config/base_settings.py
# BASE_PATH = os.path.split(os.path.split(__file__)[0])[0]
# OUTPUT_PATH = os.path.join(BASE_PATH, 'output')
# OUTPUT_GROUP_PATH = os.path.join(OUTPUT_PATH, TimeUtil.getFormatTime('%Y%m%d_%H%M%S'))
# DATABASE_NAME = os.path.join(OUTPUT_GROUP_PATH, 'Patent.db')
# EXCEL_NAME = os.path.join(OUTPUT_GROUP_PATH, '专利.xlsx')
# CHARTS_NAME = os.path.join(OUTPUT_GROUP_PATH, 'charts.html')
# LOG_FILENAME = os.path.join(OUTPUT_GROUP_PATH, "PatentCrawler.log")
# CAPTCHA_MODEL_NAME = os.path.join(BASE_PATH, 'res', 'captcha', 'sipo3.job')
# AD_PATH = os.path.join(BASE_PATH, 'res', 'advertisement', 'ad.html')
# USE_PROXY = False
# PROXY_URL = 'http://127.0.0.1:5010/get'
# TIMEOUT = 10
# DOWNLOAD_DELAY = 1
# OUTPUT_ITEMS = ['data', 'log', 'chart']
#         USE_PROXY = use_proxy
#                 PROXY_URL = proxy_url
#             TIMEOUT = timeout
#             DOWNLOAD_DELAY = delay
#             OUTPUT_ITEMS = output_items
# def check_proxy(cfg):
# def check_request(cfg: configparser.ConfigParser):
# def check_output(cfg):


from config import base_settings as bs

# Scrapy settings for crawler project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'crawler'

SPIDER_MODULES = ['crawler.spiders']
NEWSPIDER_MODULE = 'crawler.spiders'

RETRY_ENABLED = True
RETRY_TIMES = 3

# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'crawler (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs

# Path: config/base_settings.py
# BASE_PATH = os.path.split(os.path.split(__file__)[0])[0]
# OUTPUT_PATH = os.path.join(BASE_PATH, 'output')
# OUTPUT_GROUP_PATH = os.path.join(OUTPUT_PATH, TimeUtil.getFormatTime('%Y%m%d_%H%M%S'))
# DATABASE_NAME = os.path.join(OUTPUT_GROUP_PATH, 'Patent.db')
# EXCEL_NAME = os.path.join(OUTPUT_GROUP_PATH, '专利.xlsx')
# CHARTS_NAME = os.path.join(OUTPUT_GROUP_PATH, 'charts.html')
# LOG_FILENAME = os.path.join(OUTPUT_GROUP_PATH, "PatentCrawler.log")
# CAPTCHA_MODEL_NAME = os.path.join(BASE_PATH, 'res', 'captcha', 'sipo3.job')
# AD_PATH = os.path.join(BASE_PATH, 'res', 'advertisement', 'ad.html')
# USE_PROXY = False
# PROXY_URL = 'http://127.0.0.1:5010/get'
# TIMEOUT = 10
# DOWNLOAD_DELAY = 1
# OUTPUT_ITEMS = ['data', 'log', 'chart']
#         USE_PROXY = use_proxy
#                 PROXY_URL = proxy_url
#             TIMEOUT = timeout
#             DOWNLOAD_DELAY = delay
#             OUTPUT_ITEMS = output_items
# def check_proxy(cfg):
# def check_request(cfg: configparser.ConfigParser):
# def check_output(cfg):


# Path: crawler/settings.py
from config import base_settings as bs


# Scrapy settings for crawler project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     http://doc.scrapy.org/en/latest/topics/settings.html
#     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
#     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html

BOT_NAME = 'crawler'

SPIDER_MODULES = ['crawler.spiders']
NEWSPIDER_MODULE = 'crawler.spiders'

RETRY_ENABLED = True
RETRY_TIMES = 3

# Crawl responsibly by identifying yourself (and your website) on the user-agent
# USER_AGENT = 'crawler (+http://www.yourdomain.com)'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
# CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs

DOWNLOAD_DELAY = bs.DOWNLOAD_DELAY