# Path: fooltrader/api/technical.py
# def get_trading_calendar(security_type='future', exchange='shfe'):
#     the_path = get_exchange_trading_calendar_path(security_type, exchange)
# 
#     trading_dates = []
#     if os.path.exists(the_path):
#         with open(the_path) as data_file:
#             trading_dates = json.load(data_file)
#     return trading_dates
# 
# Path: fooltrader/contract/files_contract.py
# def get_exchange_trading_calendar_path(security_type='future', exchange='shfe'):
#     return os.path.join(get_exchange_dir(security_type, exchange), 'trading_calendar.json')
# 
# Path: fooltrader/utils/utils.py
# def next_date(the_time):
#     return to_timestamp(the_time) + datetime.timedelta(days=1)
# 
# def to_time_str(the_time, time_fmt=TIME_FORMAT_DAY):
#     try:
#         if time_fmt == TIME_FORMAT_MICRO:
#             return to_timestamp(the_time).strftime(time_fmt)[0:-3]
#         else:
#             return to_timestamp(the_time).strftime(time_fmt)
#     except Exception as e:
#         return the_time
# 
# def drop_duplicate(the_list):
#     return list(set(the_list))


import json
import pandas as pd
import scrapy
from datetime import datetime
from scrapy import Request
from scrapy import signals
from fooltrader.api.technical import get_trading_calendar
from fooltrader.contract.files_contract import get_exchange_trading_calendar_path
from fooltrader.utils.utils import next_date, to_time_str, drop_duplicate
        for the_date in pd.date_range(start=latest_trading_date, end=datetime.today()):
            # 双休
            if the_date.weekday() == 5 or the_date.weekday() == 6:
                continue
            the_date_str = to_time_str(the_time=the_date, time_fmt='%Y%m%d')
            yield Request(url=self.get_trading_date_url(the_date=the_date_str),
                          meta={'the_date': the_date_str},
                          callback=self.download_trading_calendar)

    def download_trading_calendar(self, response):
        if response.status == 200:
            self.trading_dates.append(response.meta['the_date'])

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(ShfeTradingCalendarSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def spider_closed(self, spider, reason):
        if self.trading_dates:
            if self.saved_trading_dates:
                self.trading_dates.append(self.saved_trading_dates)
            result_list = drop_duplicate(self.trading_dates)
            result_list = sorted(result_list)

            the_path = get_exchange_trading_calendar_path('future', 'shfe')
            with open(the_path, 'w') as outfile:
                json.dump(result_list, outfile)


# Path: fooltrader/api/technical.py
# def get_trading_calendar(security_type='future', exchange='shfe'):
#     the_path = get_exchange_trading_calendar_path(security_type, exchange)
# 
#     trading_dates = []
#     if os.path.exists(the_path):
#         with open(the_path) as data_file:
#             trading_dates = json.load(data_file)
#     return trading_dates
# 
# Path: fooltrader/contract/files_contract.py
# def get_exchange_trading_calendar_path(security_type='future', exchange='shfe'):
#     return os.path.join(get_exchange_dir(security_type, exchange), 'trading_calendar.json')
# 
# Path: fooltrader/utils/utils.py
# def next_date(the_time):
#     return to_timestamp(the_time) + datetime.timedelta(days=1)
# 
# def to_time_str(the_time, time_fmt=TIME_FORMAT_DAY):
#     try:
#         if time_fmt == TIME_FORMAT_MICRO:
#             return to_timestamp(the_time).strftime(time_fmt)[0:-3]
#         else:
#             return to_timestamp(the_time).strftime(time_fmt)
#     except Exception as e:
#         return the_time
# 
# def drop_duplicate(the_list):
#     return list(set(the_list))


# Path: fooltrader/spiders/chinafuture/shfe_trading_calendar_spider.py
import json
import pandas as pd
import scrapy
from datetime import datetime
from scrapy import Request
from scrapy import signals
from fooltrader.api.technical import get_trading_calendar
from fooltrader.contract.files_contract import get_exchange_trading_calendar_path
from fooltrader.utils.utils import next_date, to_time_str, drop_duplicate

        for the_date in pd.date_range(start=latest_trading_date, end=datetime.today()):
            # 双休
            if the_date.weekday() == 5 or the_date.weekday() == 6:
                continue
            the_date_str = to_time_str(the_time=the_date, time_fmt='%Y%m%d')
            yield Request(url=self.get_trading_date_url(the_date=the_date_str),
                          meta={'the_date': the_date_str},
                          callback=self.download_trading_calendar)

    def download_trading_calendar(self, response):
        if response.status == 200:
            self.trading_dates.append(response.meta['the_date'])

    @classmethod
    def from_crawler(cls, crawler, *args, **kwargs):
        spider = super(ShfeTradingCalendarSpider, cls).from_crawler(crawler, *args, **kwargs)
        crawler.signals.connect(spider.spider_closed, signal=signals.spider_closed)
        return spider

    def spider_closed(self, spider, reason):
        if self.trading_dates:
            if self.saved_trading_dates:
                self.trading_dates.append(self.saved_trading_dates)
            result_list = drop_duplicate(self.trading_dates)
            result_list = sorted(result_list)

            the_path = get_exchange_trading_calendar_path('future', 'shfe')
            with open(the_path, 'w') as outfile:
                json.dump(result_list, outfile)


        spider.logger.info('Spider closed: %s,%s\n', spider.name, reason)