# Path: fooltrader/contract/files_contract.py
# def get_exchange_dir(security_type='future', exchange='shfe'):
# def get_exchange_trading_calendar_path(security_type='future', exchange='shfe'):
# def get_exchange_cache_dir(security_type='future', exchange='shfe', the_year=None,
#                            data_type="day_kdata"):
# def get_exchange_cache_path(security_type='future', exchange='shfe', the_date=datetime.today(), data_type="day_kdata"):
# def get_security_list_path(security_type, exchange):
# def get_security_dir(item=None, security_type=None, exchange=None, code=None):
# def get_security_meta_path(item=None, security_type=None, exchange=None, code=None):
# def adjust_source(security_item, source):
# def get_kdata_dir(item, fuquan='bfq'):
# def get_kdata_path(item, source=None, fuquan='bfq', year=None, quarter=None):
# def get_tick_dir(item):
# def get_tick_path(item, date):
# def get_event_dir(item):
# def get_event_path(item, event_type='finance_forecast'):
# def get_finance_forecast_event_path(item):
# def get_finance_report_event_path(item):
# def get_finance_dir(item):
# def get_finance_path(item):
# def get_balance_sheet_path(item):
# def get_income_statement_path(item):
# def get_cash_flow_statement_path(item):
# def _to_valid_fuquan(fuquan='bfq'):
# def get_trading_dates_path_163(item):
# def get_trading_dates_path_ths(item):
# def get_trading_dates_path_sse(item):
# def get_code_from_path(the_path, security_type='stock'):
# 
# Path: fooltrader/contract/data_contract.py
# STOCK_META_COL = ['code', 'name', 'timestamp', 'listDate', 'exchange', 'type', 'id', 'sinaIndustry', 'sinaConcept',
#                   'sinaArea',
#                   'sector', 'industry']


import io
import os
import pandas as pd
import scrapy
from scrapy import Request
from scrapy import signals
from fooltrader.consts import DEFAULT_SH_HEADER, DEFAULT_SZ_HEADER
from fooltrader.contract import files_contract
from fooltrader.contract.data_contract import STOCK_META_COL
# -*- coding: utf-8 -*-





class ChinaStockListSpider(scrapy.Spider):
    name = "china_stock_list"

    def start_requests(self):
        yield Request(
            url='http://query.sse.com.cn/security/stock/downloadStockListFile.do?csrcCode=&stockCode=&areaName=&stockType=1',
            headers=DEFAULT_SH_HEADER,
            meta={'exchange': 'sh'},
            callback=self.download_stock_list)

        yield Request(
            url='http://www.szse.cn/szseWeb/ShowReport.szse?SHOWTYPE=xlsx&CATALOGID=1110&tab1PAGENUM=1&ENCODE=1&TABKEY=tab1',

# Path: fooltrader/contract/files_contract.py
# def get_exchange_dir(security_type='future', exchange='shfe'):
# def get_exchange_trading_calendar_path(security_type='future', exchange='shfe'):
# def get_exchange_cache_dir(security_type='future', exchange='shfe', the_year=None,
#                            data_type="day_kdata"):
# def get_exchange_cache_path(security_type='future', exchange='shfe', the_date=datetime.today(), data_type="day_kdata"):
# def get_security_list_path(security_type, exchange):
# def get_security_dir(item=None, security_type=None, exchange=None, code=None):
# def get_security_meta_path(item=None, security_type=None, exchange=None, code=None):
# def adjust_source(security_item, source):
# def get_kdata_dir(item, fuquan='bfq'):
# def get_kdata_path(item, source=None, fuquan='bfq', year=None, quarter=None):
# def get_tick_dir(item):
# def get_tick_path(item, date):
# def get_event_dir(item):
# def get_event_path(item, event_type='finance_forecast'):
# def get_finance_forecast_event_path(item):
# def get_finance_report_event_path(item):
# def get_finance_dir(item):
# def get_finance_path(item):
# def get_balance_sheet_path(item):
# def get_income_statement_path(item):
# def get_cash_flow_statement_path(item):
# def _to_valid_fuquan(fuquan='bfq'):
# def get_trading_dates_path_163(item):
# def get_trading_dates_path_ths(item):
# def get_trading_dates_path_sse(item):
# def get_code_from_path(the_path, security_type='stock'):
# 
# Path: fooltrader/contract/data_contract.py
# STOCK_META_COL = ['code', 'name', 'timestamp', 'listDate', 'exchange', 'type', 'id', 'sinaIndustry', 'sinaConcept',
#                   'sinaArea',
#                   'sector', 'industry']


# Path: fooltrader/spiders/chinastock/china_stock_list_spider.py
import io
import os
import pandas as pd
import scrapy
from scrapy import Request
from scrapy import signals
from fooltrader.consts import DEFAULT_SH_HEADER, DEFAULT_SZ_HEADER
from fooltrader.contract import files_contract
from fooltrader.contract.data_contract import STOCK_META_COL

# -*- coding: utf-8 -*-





class ChinaStockListSpider(scrapy.Spider):
    name = "china_stock_list"

    def start_requests(self):
        yield Request(
            url='http://query.sse.com.cn/security/stock/downloadStockListFile.do?csrcCode=&stockCode=&areaName=&stockType=1',
            headers=DEFAULT_SH_HEADER,
            meta={'exchange': 'sh'},
            callback=self.download_stock_list)

        yield Request(
            url='http://www.szse.cn/szseWeb/ShowReport.szse?SHOWTYPE=xlsx&CATALOGID=1110&tab1PAGENUM=1&ENCODE=1&TABKEY=tab1',

            headers=DEFAULT_SZ_HEADER,