# Path: fooltrader/api/technical.py
# def get_security_list(security_type='stock', exchanges=None, start_code=None, end_code=None,
#                       mode='simple', start_list_date=None, codes=None):
#     """
#     get security list.
# 
#     Parameters
#     ----------
#     security_type : str
#         {‘stock’, 'future'},default: stock
#     exchanges : str or list
#         ['sh', 'sz','nasdaq','nyse','amex','shfe','dce','zce'],default: ['sh','sz']
#     start_code : str
#         the start code,work with end,default:None
#         if using codes,it would be ignored
#     end_code : str
#         the end code,works with start,default:None
#         if using codes,it would be ignored
#     mode : str
#         whether parse more security info,{'simple','es'},default:'simple'
#     start_list_date : Timestamp str or Timestamp
#         the filter for start list date,default:None
#     codes : list
#         the exact codes to query,default:None
# 
#     Returns
#     -------
#     DataFrame
#         the security list
# 
#     """
#     df = pd.DataFrame()
#     if type(exchanges) == str:
#         exchanges = [exchanges]
# 
#     if not exchanges:
#         exchanges = SECURITY_TYPE_MAP_EXCHANGES[security_type]
# 
#     if security_type == 'index':
#         df = df.append(pd.DataFrame(CHINA_STOCK_SH_INDEX), ignore_index=True)
#         df = df.append(pd.DataFrame(CHINA_STOCK_SZ_INDEX), ignore_index=True)
#         df = df.append(pd.DataFrame(USA_STOCK_NASDAQ_INDEX), ignore_index=True)
#     else:
#         for exchange in exchanges:
#             the_path = get_security_list_path(security_type, exchange)
#             if os.path.exists(the_path):
#                 converters = None
# 
#                 if mode == 'es' and security_type == 'stock':
#                     converters = {'code': str,
#                                   'sinaIndustry': convert_to_list_if_need,
#                                   'sinaConcept': convert_to_list_if_need,
#                                   'sinaArea': convert_to_list_if_need}
#                 if converters:
#                     df = df.append(pd.read_csv(the_path, converters=converters), ignore_index=True)
#                 else:
#                     df = df.append(pd.read_csv(the_path, dtype=str), ignore_index=True)
# 
#     if not df.empty > 0:
#         df = df_for_date_range(df, start_date=start_list_date)
# 
#         df = df.set_index(df['code'], drop=False)
#         df = df.sort_index()
# 
#         if codes:
#             df = df[df["code"].isin(codes)]
#         elif start_code and end_code:
#             df = df[(df["code"] >= start_code) & (df["code"] <= end_code)]
# 
#         if security_type != 'cryptocurrency':
#             df = df.drop_duplicates(subset='code', keep='last')
# 
#     return df
# 
# Path: fooltrader/contract/data_contract.py
# KDATA_COLUMN_163 = ['timestamp', 'code', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
#                     'preClose', 'change', 'changePct', 'turnoverRate', 'tCap', 'mCap']
# 
# KDATA_INDEX_COLUMN_163 = ['timestamp', 'code', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
#                           'preClose', 'change', 'changePct']
# 
# KDATA_INDEX_COL = ['timestamp', 'code', 'name', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
#                    'preClose', 'change', 'changePct', 'turnoverRate', 'tCap', 'mCap', 'pe']
# 
# KDATA_STOCK_COL = ['timestamp', 'code', 'name', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
#                    'preClose', 'change', 'changePct', 'turnoverRate', 'tCap', 'mCap', 'factor']
# 
# Path: fooltrader/contract/files_contract.py
# def get_kdata_path(item, source=None, fuquan='bfq', year=None, quarter=None):
#     source = adjust_source(item, source)
#     if source == 'sina':
#         if not year and not quarter:
#             return os.path.join(get_kdata_dir(item, fuquan), 'dayk.csv')
#         else:
#             return os.path.join(get_kdata_dir(item, fuquan), '{}Q{}.csv'.format(year, quarter))
#     else:
#         return os.path.join(get_kdata_dir(item, fuquan), '{}_dayk.csv'.format(source))
# 
# Path: fooltrader/settings.py
# US_STOCK_CODES = ['MSFT', 'AMZN', 'NVDA', 'SINA', 'AAPL']
# 
# Path: fooltrader/utils/utils.py
# def to_time_str(the_time, time_fmt=TIME_FORMAT_DAY):
#     try:
#         if time_fmt == TIME_FORMAT_MICRO:
#             return to_timestamp(the_time).strftime(time_fmt)[0:-3]
#         else:
#             return to_timestamp(the_time).strftime(time_fmt)
#     except Exception as e:
#         return the_time


import json
import os
import pandas as pd
import scrapy
from scrapy import Request
from scrapy import signals
from fooltrader.api.technical import get_security_list
from fooltrader.contract.data_contract import KDATA_COLUMN_163, KDATA_INDEX_COLUMN_163, \
    KDATA_INDEX_COL, KDATA_STOCK_COL
from fooltrader.contract.files_contract import get_kdata_path
from fooltrader.settings import US_STOCK_CODES
from fooltrader.utils.utils import to_time_str
# -*- coding: utf-8 -*-





class AmericaStockKdataSpider(scrapy.Spider):
    name = "americ_stock_kdata"

    custom_settings = {
        # 'DOWNLOAD_DELAY': 2,
        # 'CONCURRENT_REQUESTS_PER_DOMAIN': 8,

        'SPIDER_MIDDLEWARES': {
            'fooltrader.middlewares.FoolErrorMiddleware': 1000,
        }

# Path: fooltrader/api/technical.py
# def get_security_list(security_type='stock', exchanges=None, start_code=None, end_code=None,
#                       mode='simple', start_list_date=None, codes=None):
#     """
#     get security list.
# 
#     Parameters
#     ----------
#     security_type : str
#         {‘stock’, 'future'},default: stock
#     exchanges : str or list
#         ['sh', 'sz','nasdaq','nyse','amex','shfe','dce','zce'],default: ['sh','sz']
#     start_code : str
#         the start code,work with end,default:None
#         if using codes,it would be ignored
#     end_code : str
#         the end code,works with start,default:None
#         if using codes,it would be ignored
#     mode : str
#         whether parse more security info,{'simple','es'},default:'simple'
#     start_list_date : Timestamp str or Timestamp
#         the filter for start list date,default:None
#     codes : list
#         the exact codes to query,default:None
# 
#     Returns
#     -------
#     DataFrame
#         the security list
# 
#     """
#     df = pd.DataFrame()
#     if type(exchanges) == str:
#         exchanges = [exchanges]
# 
#     if not exchanges:
#         exchanges = SECURITY_TYPE_MAP_EXCHANGES[security_type]
# 
#     if security_type == 'index':
#         df = df.append(pd.DataFrame(CHINA_STOCK_SH_INDEX), ignore_index=True)
#         df = df.append(pd.DataFrame(CHINA_STOCK_SZ_INDEX), ignore_index=True)
#         df = df.append(pd.DataFrame(USA_STOCK_NASDAQ_INDEX), ignore_index=True)
#     else:
#         for exchange in exchanges:
#             the_path = get_security_list_path(security_type, exchange)
#             if os.path.exists(the_path):
#                 converters = None
# 
#                 if mode == 'es' and security_type == 'stock':
#                     converters = {'code': str,
#                                   'sinaIndustry': convert_to_list_if_need,
#                                   'sinaConcept': convert_to_list_if_need,
#                                   'sinaArea': convert_to_list_if_need}
#                 if converters:
#                     df = df.append(pd.read_csv(the_path, converters=converters), ignore_index=True)
#                 else:
#                     df = df.append(pd.read_csv(the_path, dtype=str), ignore_index=True)
# 
#     if not df.empty > 0:
#         df = df_for_date_range(df, start_date=start_list_date)
# 
#         df = df.set_index(df['code'], drop=False)
#         df = df.sort_index()
# 
#         if codes:
#             df = df[df["code"].isin(codes)]
#         elif start_code and end_code:
#             df = df[(df["code"] >= start_code) & (df["code"] <= end_code)]
# 
#         if security_type != 'cryptocurrency':
#             df = df.drop_duplicates(subset='code', keep='last')
# 
#     return df
# 
# Path: fooltrader/contract/data_contract.py
# KDATA_COLUMN_163 = ['timestamp', 'code', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
#                     'preClose', 'change', 'changePct', 'turnoverRate', 'tCap', 'mCap']
# 
# KDATA_INDEX_COLUMN_163 = ['timestamp', 'code', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
#                           'preClose', 'change', 'changePct']
# 
# KDATA_INDEX_COL = ['timestamp', 'code', 'name', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
#                    'preClose', 'change', 'changePct', 'turnoverRate', 'tCap', 'mCap', 'pe']
# 
# KDATA_STOCK_COL = ['timestamp', 'code', 'name', 'low', 'open', 'close', 'high', 'volume', 'turnover', 'securityId',
#                    'preClose', 'change', 'changePct', 'turnoverRate', 'tCap', 'mCap', 'factor']
# 
# Path: fooltrader/contract/files_contract.py
# def get_kdata_path(item, source=None, fuquan='bfq', year=None, quarter=None):
#     source = adjust_source(item, source)
#     if source == 'sina':
#         if not year and not quarter:
#             return os.path.join(get_kdata_dir(item, fuquan), 'dayk.csv')
#         else:
#             return os.path.join(get_kdata_dir(item, fuquan), '{}Q{}.csv'.format(year, quarter))
#     else:
#         return os.path.join(get_kdata_dir(item, fuquan), '{}_dayk.csv'.format(source))
# 
# Path: fooltrader/settings.py
# US_STOCK_CODES = ['MSFT', 'AMZN', 'NVDA', 'SINA', 'AAPL']
# 
# Path: fooltrader/utils/utils.py
# def to_time_str(the_time, time_fmt=TIME_FORMAT_DAY):
#     try:
#         if time_fmt == TIME_FORMAT_MICRO:
#             return to_timestamp(the_time).strftime(time_fmt)[0:-3]
#         else:
#             return to_timestamp(the_time).strftime(time_fmt)
#     except Exception as e:
#         return the_time


# Path: fooltrader/spiders/america/america_stock_kdata_spider_163.py
import json
import os
import pandas as pd
import scrapy
from scrapy import Request
from scrapy import signals
from fooltrader.api.technical import get_security_list
from fooltrader.contract.data_contract import KDATA_COLUMN_163, KDATA_INDEX_COLUMN_163, \
    KDATA_INDEX_COL, KDATA_STOCK_COL
from fooltrader.contract.files_contract import get_kdata_path
from fooltrader.settings import US_STOCK_CODES
from fooltrader.utils.utils import to_time_str

# -*- coding: utf-8 -*-





class AmericaStockKdataSpider(scrapy.Spider):
    name = "americ_stock_kdata"

    custom_settings = {
        # 'DOWNLOAD_DELAY': 2,
        # 'CONCURRENT_REQUESTS_PER_DOMAIN': 8,

        'SPIDER_MIDDLEWARES': {
            'fooltrader.middlewares.FoolErrorMiddleware': 1000,
        }

    }