# Path: kquant_data/config.py
# __CONFIG_H5_STK_DIR__ = r'D:\DATA_STK'
# 
# __CONFIG_H5_STK_DIVIDEND_DIR__ = os.path.join(__CONFIG_H5_STK_DIR__, 'dividend')
# 
# Path: kquant_data/stock/stock.py
# def sort_dividend(divs):
# def factor(daily, divs, ndigits):
# def adjust(df, adjust_type=None):
# def merge_adjust_factor(df, div):
# def read_h5_tdx(market, code, bar_size, h5_path, tdx_path, div_path):
# def _export_dividend_from_data(tdx_root, dividend_output, daily_output, data):
# def export_dividend_daily_dzh(dzh_input, tdx_root, dividend_output, daily_output):
# def export_dividend_daily_gbbq(gbbq_input, tdx_root, dividend_output, daily_output):
# 
# Path: kquant_data/processing/utils.py
# def filter_dataframe(df, index_name=None, start_date=None, end_date=None, fields=None):
#     if index_name is not None:
#         df['index_datetime'] = df[index_name].apply(yyyyMMddHHmm_2_datetime)
#         df = df.set_index('index_datetime')
#     # 过滤时间
#     if start_date is not None or end_date is not None:
#         df = df[start_date:end_date]
#     # 过滤字段
#     if fields is not None:
#         df = df[fields]
#     return df
# 
# def multiprocessing_convert(multi, rule, _input, output, instruments, func_convert):
#     tic()
# 
#     if multi:
#         pool_size = multiprocessing.cpu_count() - 1
#         pool = multiprocessing.Pool(processes=pool_size)
#         func = partial(func_convert, rule, _input, output, instruments)
#         pool_outputs = pool.map(func, range(len(instruments)))
#         print('Pool:', pool_outputs)
#     else:
#         for i in range(len(instruments)):
#             func_convert(rule, _input, output, instruments, i)
# 
#     toc()
# 
# Path: kquant_data/stock/symbol.py
# def get_folder_symbols(folder, sub_folder):
#     path = os.path.join(folder, sub_folder, 'sh')
#     df_sh = get_symbols_from_path(path, "SSE")
#     path = os.path.join(folder, sub_folder, 'sz')
#     df_sz = get_symbols_from_path(path, "SZSE")
#     df = pd.concat([df_sh, df_sz])
# 
#     return df


import os
import pandas as pd
from kquant_data.config import __CONFIG_H5_STK_DIR__, __CONFIG_H5_STK_DIVIDEND_DIR__
from kquant_data.stock.stock import merge_adjust_factor, bars_to_h5
from kquant_data.processing.utils import filter_dataframe, multiprocessing_convert
from kquant_data.stock.symbol import get_folder_symbols
            df = df_new
        else:
            df_old = filter_dataframe(df_old, 'DateTime', None, None, None)
            # 数据合并，不能简单的合并
            # 需要保留老的，新的重复的地方忽略
            last_ts = df_old.index[-1]
            df_new2 = df_new[last_ts:][1:]

            df = pd.concat([df_old, df_new2])
    except:
        df = df_new

    # 有可能没有除权文件
    div_path = os.path.join(__CONFIG_H5_STK_DIVIDEND_DIR__, "%s.h5" % t['local_symbol'])
    try:
        div = pd.read_hdf(div_path)
        div = filter_dataframe(div, 'time', None, None, None)
        df = merge_adjust_factor(df, div)
    except:
        # 这里一般是文件没找到，表示没有除权信息
        df['backward_factor'] = 1
        df['forward_factor'] = 1

    bars_to_h5(path_old, df)


if __name__ == '__main__':
    # 此合并h5的代码已经废弃不用
    _input = '5min_lc5'
    _ouput = '5min'

# Path: kquant_data/config.py
# __CONFIG_H5_STK_DIR__ = r'D:\DATA_STK'
# 
# __CONFIG_H5_STK_DIVIDEND_DIR__ = os.path.join(__CONFIG_H5_STK_DIR__, 'dividend')
# 
# Path: kquant_data/stock/stock.py
# def sort_dividend(divs):
# def factor(daily, divs, ndigits):
# def adjust(df, adjust_type=None):
# def merge_adjust_factor(df, div):
# def read_h5_tdx(market, code, bar_size, h5_path, tdx_path, div_path):
# def _export_dividend_from_data(tdx_root, dividend_output, daily_output, data):
# def export_dividend_daily_dzh(dzh_input, tdx_root, dividend_output, daily_output):
# def export_dividend_daily_gbbq(gbbq_input, tdx_root, dividend_output, daily_output):
# 
# Path: kquant_data/processing/utils.py
# def filter_dataframe(df, index_name=None, start_date=None, end_date=None, fields=None):
#     if index_name is not None:
#         df['index_datetime'] = df[index_name].apply(yyyyMMddHHmm_2_datetime)
#         df = df.set_index('index_datetime')
#     # 过滤时间
#     if start_date is not None or end_date is not None:
#         df = df[start_date:end_date]
#     # 过滤字段
#     if fields is not None:
#         df = df[fields]
#     return df
# 
# def multiprocessing_convert(multi, rule, _input, output, instruments, func_convert):
#     tic()
# 
#     if multi:
#         pool_size = multiprocessing.cpu_count() - 1
#         pool = multiprocessing.Pool(processes=pool_size)
#         func = partial(func_convert, rule, _input, output, instruments)
#         pool_outputs = pool.map(func, range(len(instruments)))
#         print('Pool:', pool_outputs)
#     else:
#         for i in range(len(instruments)):
#             func_convert(rule, _input, output, instruments, i)
# 
#     toc()
# 
# Path: kquant_data/stock/symbol.py
# def get_folder_symbols(folder, sub_folder):
#     path = os.path.join(folder, sub_folder, 'sh')
#     df_sh = get_symbols_from_path(path, "SSE")
#     path = os.path.join(folder, sub_folder, 'sz')
#     df_sz = get_symbols_from_path(path, "SZSE")
#     df = pd.concat([df_sh, df_sz])
# 
#     return df


# Path: demo_stock/B_5min/A02_concat_h5.py
import os
import pandas as pd
from kquant_data.config import __CONFIG_H5_STK_DIR__, __CONFIG_H5_STK_DIVIDEND_DIR__
from kquant_data.stock.stock import merge_adjust_factor, bars_to_h5
from kquant_data.processing.utils import filter_dataframe, multiprocessing_convert
from kquant_data.stock.symbol import get_folder_symbols

            df = df_new
        else:
            df_old = filter_dataframe(df_old, 'DateTime', None, None, None)
            # 数据合并，不能简单的合并
            # 需要保留老的，新的重复的地方忽略
            last_ts = df_old.index[-1]
            df_new2 = df_new[last_ts:][1:]

            df = pd.concat([df_old, df_new2])
    except:
        df = df_new

    # 有可能没有除权文件
    div_path = os.path.join(__CONFIG_H5_STK_DIVIDEND_DIR__, "%s.h5" % t['local_symbol'])
    try:
        div = pd.read_hdf(div_path)
        div = filter_dataframe(div, 'time', None, None, None)
        df = merge_adjust_factor(df, div)
    except:
        # 这里一般是文件没找到，表示没有除权信息
        df['backward_factor'] = 1
        df['forward_factor'] = 1

    bars_to_h5(path_old, df)


if __name__ == '__main__':
    # 此合并h5的代码已经废弃不用
    _input = '5min_lc5'
    _ouput = '5min'

    instruments = get_folder_symbols(__CONFIG_H5_STK_DIR__, _input)