# Path: fooltrader/api/technical.py
# def get_trading_calendar(security_type='future', exchange='shfe'):
#     the_path = get_exchange_trading_calendar_path(security_type, exchange)
# 
#     trading_dates = []
#     if os.path.exists(the_path):
#         with open(the_path) as data_file:
#             trading_dates = json.load(data_file)
#     return trading_dates
# 
# Path: fooltrader/contract/files_contract.py
# def get_exchange_trading_calendar_path(security_type='future', exchange='shfe'):
#     return os.path.join(get_exchange_dir(security_type, exchange), 'trading_calendar.json')
# 
# Path: fooltrader/utils/utils.py
# def next_date(the_time):
#     return to_timestamp(the_time) + datetime.timedelta(days=1)
# 
# def to_time_str(the_time, time_fmt=TIME_FORMAT_DAY):
#     try:
#         if time_fmt == TIME_FORMAT_MICRO:
#             return to_timestamp(the_time).strftime(time_fmt)[0:-3]
#         else:
#             return to_timestamp(the_time).strftime(time_fmt)
#     except Exception as e:
#         return the_time
# 
# def drop_duplicate(the_list):
#     return list(set(the_list))


import json
import pandas as pd
import scrapy
from datetime import datetime
from scrapy import Request
from scrapy import signals
from fooltrader.api.technical import get_trading_calendar
from fooltrader.contract.files_contract import get_exchange_trading_calendar_path
from fooltrader.utils.utils import next_date, to_time_str, drop_duplicate

        # 'SPIDER_MIDDLEWARES': {
        #     'fooltrader.middlewares.FoolErrorMiddleware': 1000,
        # }
    }

    def __init__(self, name=None, **kwargs):
        super().__init__(name, **kwargs)
        # python list  thread safe,no need lock
        self.saved_trading_dates = get_trading_calendar(security_type='future', exchange='shfe')
        self.trading_dates = []

    def start_requests(self):
        # 往年的统计数据可以直接下载，只从今年开始
        latest_trading_date = "{}0101".format(datetime.today().year)

        # 检查已经保存的
        if self.saved_trading_dates:
            latest_trading_date = self.saved_trading_dates[-1]
            latest_trading_date = next_date(latest_trading_date)

        for the_date in pd.date_range(start=latest_trading_date, end=datetime.today()):
            # 双休
            if the_date.weekday() == 5 or the_date.weekday() == 6:
                continue
            the_date_str = to_time_str(the_time=the_date, time_fmt='%Y%m%d')
            yield Request(url=self.get_trading_date_url(the_date=the_date_str),
                          meta={'the_date': the_date_str},
                          callback=self.download_trading_calendar)


# Path: fooltrader/api/technical.py
# def get_trading_calendar(security_type='future', exchange='shfe'):
#     the_path = get_exchange_trading_calendar_path(security_type, exchange)
# 
#     trading_dates = []
#     if os.path.exists(the_path):
#         with open(the_path) as data_file:
#             trading_dates = json.load(data_file)
#     return trading_dates
# 
# Path: fooltrader/contract/files_contract.py
# def get_exchange_trading_calendar_path(security_type='future', exchange='shfe'):
#     return os.path.join(get_exchange_dir(security_type, exchange), 'trading_calendar.json')
# 
# Path: fooltrader/utils/utils.py
# def next_date(the_time):
#     return to_timestamp(the_time) + datetime.timedelta(days=1)
# 
# def to_time_str(the_time, time_fmt=TIME_FORMAT_DAY):
#     try:
#         if time_fmt == TIME_FORMAT_MICRO:
#             return to_timestamp(the_time).strftime(time_fmt)[0:-3]
#         else:
#             return to_timestamp(the_time).strftime(time_fmt)
#     except Exception as e:
#         return the_time
# 
# def drop_duplicate(the_list):
#     return list(set(the_list))


# Path: fooltrader/spiders/chinafuture/shfe_trading_calendar_spider.py
import json
import pandas as pd
import scrapy
from datetime import datetime
from scrapy import Request
from scrapy import signals
from fooltrader.api.technical import get_trading_calendar
from fooltrader.contract.files_contract import get_exchange_trading_calendar_path
from fooltrader.utils.utils import next_date, to_time_str, drop_duplicate


        # 'SPIDER_MIDDLEWARES': {
        #     'fooltrader.middlewares.FoolErrorMiddleware': 1000,
        # }
    }

    def __init__(self, name=None, **kwargs):
        super().__init__(name, **kwargs)
        # python list  thread safe,no need lock
        self.saved_trading_dates = get_trading_calendar(security_type='future', exchange='shfe')
        self.trading_dates = []

    def start_requests(self):
        # 往年的统计数据可以直接下载，只从今年开始
        latest_trading_date = "{}0101".format(datetime.today().year)

        # 检查已经保存的
        if self.saved_trading_dates:
            latest_trading_date = self.saved_trading_dates[-1]
            latest_trading_date = next_date(latest_trading_date)

        for the_date in pd.date_range(start=latest_trading_date, end=datetime.today()):
            # 双休
            if the_date.weekday() == 5 or the_date.weekday() == 6:
                continue
            the_date_str = to_time_str(the_time=the_date, time_fmt='%Y%m%d')
            yield Request(url=self.get_trading_date_url(the_date=the_date_str),
                          meta={'the_date': the_date_str},
                          callback=self.download_trading_calendar)


    def download_trading_calendar(self, response):