# Path: kquant_data/config.py
# __CONFIG_H5_STK_DIR__ = r'D:\DATA_STK'
# 
# __CONFIG_TDX_STK_DIR__ = r'D:\new_hbzq'
# 
# Path: kquant_data/stock/symbol.py
# def get_symbols_from_path_only_stock(path, exchange):
#     """
#     指定目录，将目录转成合约列表
#     :param path:
#     :param exchange:
#     :return:
#     """
#     list1 = []
#     list2 = []
#     list3 = []
#     list4 = []
#     list5 = []
#     for dirpath, dirnames, filenames in os.walk(path):
#         for filename in filenames:
#             if is_stock(filename):
#                 list1.append(filename[:8])
#                 list2.append(filename[:2])
#                 list3.append(filename[2:8])
#                 list4.append("%s.%s" % (filename[2:8], exchange))
#                 list5.append("%s.%s" % (filename[2:8], filename[:2].upper()))
# 
#     df = pd.DataFrame({"local_symbol": list1, "market": list2, "code": list3, "symbol": list4, "wind_code": list5})
# 
#     return df
# 
# Path: kquant_data/stock/tdx.py
# def get_tdx_path(market, code, bar_size):
#     # D:\new_hbzq\vipdoc\sh\lday\sh000001.day
#     # D:\new_hbzq\vipdoc\sh\fzline\sh000001.lc5
#     # D:\new_hbzq\vipdoc\sh\minline\sh000001.lc1
#     folder = bar_size_2_folder(bar_size)
#     file_ext = bar_size_2_file_ext(bar_size)
#     filename = "%s%s.%s" % (market, code, file_ext)
#     return os.path.join("vipdoc", market, folder, filename)
# 
# def bars_to_h5(input_path, data):  # 保存日线
#     write_dataframe_set_struct_keep_head(input_path, data, tdx_h5_type, 'BarData')
#     return
# 
# Path: kquant_data/stock/stock.py
# def sort_dividend(divs):
# def factor(daily, divs, ndigits):
# def adjust(df, adjust_type=None):
# def merge_adjust_factor(df, div):
# def read_h5_tdx(market, code, bar_size, h5_path, tdx_path, div_path):
# def _export_dividend_from_data(tdx_root, dividend_output, daily_output, data):
# def export_dividend_daily_dzh(dzh_input, tdx_root, dividend_output, daily_output):
# def export_dividend_daily_gbbq(gbbq_input, tdx_root, dividend_output, daily_output):
# 
# Path: kquant_data/processing/utils.py
# def multiprocessing_convert(multi, rule, _input, output, instruments, func_convert):
#     tic()
# 
#     if multi:
#         pool_size = multiprocessing.cpu_count() - 1
#         pool = multiprocessing.Pool(processes=pool_size)
#         func = partial(func_convert, rule, _input, output, instruments)
#         pool_outputs = pool.map(func, range(len(instruments)))
#         print('Pool:', pool_outputs)
#     else:
#         for i in range(len(instruments)):
#             func_convert(rule, _input, output, instruments, i)
# 
#     toc()


import os
import pandas as pd
from kquant_data.config import __CONFIG_H5_STK_DIR__, __CONFIG_TDX_STK_DIR__
from kquant_data.stock.symbol import get_symbols_from_path_only_stock
from kquant_data.stock.tdx import get_tdx_path, bars_to_h5
from kquant_data.stock.stock import read_file
from kquant_data.processing.utils import multiprocessing_convert
http://www.tdx.com.cn/list_66_69.html

建立与fzline同级目录的5文件夹，将数据复制进去
运行当前程序，并转换到5min_5
现在5min_lc5是最新的数据，只要将5min_5中的数据复制到5min中，然后执行合并的脚本即可
"""



def _export_data(rule, _input, output, instruments, i):
    t = instruments.iloc[i]
    print("%d %s" % (i, t['local_symbol']))

    _tdx_path = os.path.join(__CONFIG_TDX_STK_DIR__, get_tdx_path(t['market'], t['code'], 5))
    try:
        df = read_file(_tdx_path)
    except FileNotFoundError:
        # 没有原始的数据文件
        return None

    # 导出到临时目录时因子都用1
    df['backward_factor'] = 1
    df['forward_factor'] = 1
    data_output = os.path.join(__CONFIG_H5_STK_DIR__, output, t['market'], "%s.h5" % t['local_symbol'])
    bars_to_h5(data_output, df)


def export_symbols():
    path = os.path.join(__CONFIG_TDX_STK_DIR__, "vipdoc", 'sh', '5')
    df_sh = get_symbols_from_path_only_stock(path, "SSE")

# Path: kquant_data/config.py
# __CONFIG_H5_STK_DIR__ = r'D:\DATA_STK'
# 
# __CONFIG_TDX_STK_DIR__ = r'D:\new_hbzq'
# 
# Path: kquant_data/stock/symbol.py
# def get_symbols_from_path_only_stock(path, exchange):
#     """
#     指定目录，将目录转成合约列表
#     :param path:
#     :param exchange:
#     :return:
#     """
#     list1 = []
#     list2 = []
#     list3 = []
#     list4 = []
#     list5 = []
#     for dirpath, dirnames, filenames in os.walk(path):
#         for filename in filenames:
#             if is_stock(filename):
#                 list1.append(filename[:8])
#                 list2.append(filename[:2])
#                 list3.append(filename[2:8])
#                 list4.append("%s.%s" % (filename[2:8], exchange))
#                 list5.append("%s.%s" % (filename[2:8], filename[:2].upper()))
# 
#     df = pd.DataFrame({"local_symbol": list1, "market": list2, "code": list3, "symbol": list4, "wind_code": list5})
# 
#     return df
# 
# Path: kquant_data/stock/tdx.py
# def get_tdx_path(market, code, bar_size):
#     # D:\new_hbzq\vipdoc\sh\lday\sh000001.day
#     # D:\new_hbzq\vipdoc\sh\fzline\sh000001.lc5
#     # D:\new_hbzq\vipdoc\sh\minline\sh000001.lc1
#     folder = bar_size_2_folder(bar_size)
#     file_ext = bar_size_2_file_ext(bar_size)
#     filename = "%s%s.%s" % (market, code, file_ext)
#     return os.path.join("vipdoc", market, folder, filename)
# 
# def bars_to_h5(input_path, data):  # 保存日线
#     write_dataframe_set_struct_keep_head(input_path, data, tdx_h5_type, 'BarData')
#     return
# 
# Path: kquant_data/stock/stock.py
# def sort_dividend(divs):
# def factor(daily, divs, ndigits):
# def adjust(df, adjust_type=None):
# def merge_adjust_factor(df, div):
# def read_h5_tdx(market, code, bar_size, h5_path, tdx_path, div_path):
# def _export_dividend_from_data(tdx_root, dividend_output, daily_output, data):
# def export_dividend_daily_dzh(dzh_input, tdx_root, dividend_output, daily_output):
# def export_dividend_daily_gbbq(gbbq_input, tdx_root, dividend_output, daily_output):
# 
# Path: kquant_data/processing/utils.py
# def multiprocessing_convert(multi, rule, _input, output, instruments, func_convert):
#     tic()
# 
#     if multi:
#         pool_size = multiprocessing.cpu_count() - 1
#         pool = multiprocessing.Pool(processes=pool_size)
#         func = partial(func_convert, rule, _input, output, instruments)
#         pool_outputs = pool.map(func, range(len(instruments)))
#         print('Pool:', pool_outputs)
#     else:
#         for i in range(len(instruments)):
#             func_convert(rule, _input, output, instruments, i)
# 
#     toc()


# Path: demo_stock/B_5min/B01_export_5.py
import os
import pandas as pd
from kquant_data.config import __CONFIG_H5_STK_DIR__, __CONFIG_TDX_STK_DIR__
from kquant_data.stock.symbol import get_symbols_from_path_only_stock
from kquant_data.stock.tdx import get_tdx_path, bars_to_h5
from kquant_data.stock.stock import read_file
from kquant_data.processing.utils import multiprocessing_convert

http://www.tdx.com.cn/list_66_69.html

建立与fzline同级目录的5文件夹，将数据复制进去
运行当前程序，并转换到5min_5
现在5min_lc5是最新的数据，只要将5min_5中的数据复制到5min中，然后执行合并的脚本即可
"""



def _export_data(rule, _input, output, instruments, i):
    t = instruments.iloc[i]
    print("%d %s" % (i, t['local_symbol']))

    _tdx_path = os.path.join(__CONFIG_TDX_STK_DIR__, get_tdx_path(t['market'], t['code'], 5))
    try:
        df = read_file(_tdx_path)
    except FileNotFoundError:
        # 没有原始的数据文件
        return None

    # 导出到临时目录时因子都用1
    df['backward_factor'] = 1
    df['forward_factor'] = 1
    data_output = os.path.join(__CONFIG_H5_STK_DIR__, output, t['market'], "%s.h5" % t['local_symbol'])
    bars_to_h5(data_output, df)


def export_symbols():
    path = os.path.join(__CONFIG_TDX_STK_DIR__, "vipdoc", 'sh', '5')
    df_sh = get_symbols_from_path_only_stock(path, "SSE")

    path = os.path.join(__CONFIG_TDX_STK_DIR__, "vipdoc", 'sz', '5')