# Path: fooltrader/contract/files_contract.py
# def get_exchange_dir(security_type='future', exchange='shfe'):
# def get_exchange_trading_calendar_path(security_type='future', exchange='shfe'):
# def get_exchange_cache_dir(security_type='future', exchange='shfe', the_year=None,
#                            data_type="day_kdata"):
# def get_exchange_cache_path(security_type='future', exchange='shfe', the_date=datetime.today(), data_type="day_kdata"):
# def get_security_list_path(security_type, exchange):
# def get_security_dir(item=None, security_type=None, exchange=None, code=None):
# def get_security_meta_path(item=None, security_type=None, exchange=None, code=None):
# def adjust_source(security_item, source):
# def get_kdata_dir(item, fuquan='bfq'):
# def get_kdata_path(item, source=None, fuquan='bfq', year=None, quarter=None):
# def get_tick_dir(item):
# def get_tick_path(item, date):
# def get_event_dir(item):
# def get_event_path(item, event_type='finance_forecast'):
# def get_finance_forecast_event_path(item):
# def get_finance_report_event_path(item):
# def get_finance_dir(item):
# def get_finance_path(item):
# def get_balance_sheet_path(item):
# def get_income_statement_path(item):
# def get_cash_flow_statement_path(item):
# def _to_valid_fuquan(fuquan='bfq'):
# def get_trading_dates_path_163(item):
# def get_trading_dates_path_ths(item):
# def get_trading_dates_path_sse(item):
# def get_code_from_path(the_path, security_type='stock'):
# 
# Path: fooltrader/contract/data_contract.py
# STOCK_META_COL = ['code', 'name', 'timestamp', 'listDate', 'exchange', 'type', 'id', 'sinaIndustry', 'sinaConcept',
#                   'sinaArea',
#                   'sector', 'industry']


import io
import os
import pandas as pd
import scrapy
from scrapy import Request
from scrapy import signals
from fooltrader.consts import DEFAULT_SH_HEADER, DEFAULT_SZ_HEADER
from fooltrader.contract import files_contract
from fooltrader.contract.data_contract import STOCK_META_COL
        exchange = response.meta['exchange']
        path = files_contract.get_security_list_path('stock', exchange)
        df = None
        if exchange == 'sh':
            df = pd.read_csv(io.BytesIO(response.body), sep='\s+', encoding='GB2312', dtype=str)
        elif exchange == 'sz':
            df = pd.read_excel(io.BytesIO(response.body), sheet_name='上市公司列表', dtype=str)
        if df is not None:
            if os.path.exists(path):
                df_current = pd.read_csv(path, dtype=str)
                df_current = df_current.set_index('code', drop=False)
            else:
                df_current = pd.DataFrame()

            df = df.loc[:, ['A股代码', 'A股简称', 'A股上市日期']]
            df.columns = ['code', 'name', 'listDate']
            df['exchange'] = exchange
            df['type'] = 'stock'
            df['id'] = df[['type', 'exchange', 'code']].apply(lambda x: '_'.join(x.astype(str)), axis=1)
            df['timestamp'] = df['listDate']
            df = df.dropna(axis=0, how='any')
            df = df.set_index('code', drop=False)

            # 只添加增量
            diff = set(df.index.tolist()) - set(df_current.index.tolist())
            diff = [item for item in diff if item != 'nan']

            if diff:
                df_current = df_current.append(df.loc[diff, :], ignore_index=False)
                df_current = df_current.loc[:, STOCK_META_COL]

# Path: fooltrader/contract/files_contract.py
# def get_exchange_dir(security_type='future', exchange='shfe'):
# def get_exchange_trading_calendar_path(security_type='future', exchange='shfe'):
# def get_exchange_cache_dir(security_type='future', exchange='shfe', the_year=None,
#                            data_type="day_kdata"):
# def get_exchange_cache_path(security_type='future', exchange='shfe', the_date=datetime.today(), data_type="day_kdata"):
# def get_security_list_path(security_type, exchange):
# def get_security_dir(item=None, security_type=None, exchange=None, code=None):
# def get_security_meta_path(item=None, security_type=None, exchange=None, code=None):
# def adjust_source(security_item, source):
# def get_kdata_dir(item, fuquan='bfq'):
# def get_kdata_path(item, source=None, fuquan='bfq', year=None, quarter=None):
# def get_tick_dir(item):
# def get_tick_path(item, date):
# def get_event_dir(item):
# def get_event_path(item, event_type='finance_forecast'):
# def get_finance_forecast_event_path(item):
# def get_finance_report_event_path(item):
# def get_finance_dir(item):
# def get_finance_path(item):
# def get_balance_sheet_path(item):
# def get_income_statement_path(item):
# def get_cash_flow_statement_path(item):
# def _to_valid_fuquan(fuquan='bfq'):
# def get_trading_dates_path_163(item):
# def get_trading_dates_path_ths(item):
# def get_trading_dates_path_sse(item):
# def get_code_from_path(the_path, security_type='stock'):
# 
# Path: fooltrader/contract/data_contract.py
# STOCK_META_COL = ['code', 'name', 'timestamp', 'listDate', 'exchange', 'type', 'id', 'sinaIndustry', 'sinaConcept',
#                   'sinaArea',
#                   'sector', 'industry']


# Path: fooltrader/spiders/chinastock/china_stock_list_spider.py
import io
import os
import pandas as pd
import scrapy
from scrapy import Request
from scrapy import signals
from fooltrader.consts import DEFAULT_SH_HEADER, DEFAULT_SZ_HEADER
from fooltrader.contract import files_contract
from fooltrader.contract.data_contract import STOCK_META_COL

        exchange = response.meta['exchange']
        path = files_contract.get_security_list_path('stock', exchange)
        df = None
        if exchange == 'sh':
            df = pd.read_csv(io.BytesIO(response.body), sep='\s+', encoding='GB2312', dtype=str)
        elif exchange == 'sz':
            df = pd.read_excel(io.BytesIO(response.body), sheet_name='上市公司列表', dtype=str)
        if df is not None:
            if os.path.exists(path):
                df_current = pd.read_csv(path, dtype=str)
                df_current = df_current.set_index('code', drop=False)
            else:
                df_current = pd.DataFrame()

            df = df.loc[:, ['A股代码', 'A股简称', 'A股上市日期']]
            df.columns = ['code', 'name', 'listDate']
            df['exchange'] = exchange
            df['type'] = 'stock'
            df['id'] = df[['type', 'exchange', 'code']].apply(lambda x: '_'.join(x.astype(str)), axis=1)
            df['timestamp'] = df['listDate']
            df = df.dropna(axis=0, how='any')
            df = df.set_index('code', drop=False)

            # 只添加增量
            diff = set(df.index.tolist()) - set(df_current.index.tolist())
            diff = [item for item in diff if item != 'nan']

            if diff:
                df_current = df_current.append(df.loc[diff, :], ignore_index=False)
                df_current = df_current.loc[:, STOCK_META_COL]

                df_current.columns = STOCK_META_COL