# Path: kquant_data/config.py
# __CONFIG_H5_STK_DIR__ = r'D:\DATA_STK'
# 
# Path: kquant_data/stock/stock.py
# def sort_dividend(divs):
# def factor(daily, divs, ndigits):
# def adjust(df, adjust_type=None):
# def merge_adjust_factor(df, div):
# def read_h5_tdx(market, code, bar_size, h5_path, tdx_path, div_path):
# def _export_dividend_from_data(tdx_root, dividend_output, daily_output, data):
# def export_dividend_daily_dzh(dzh_input, tdx_root, dividend_output, daily_output):
# def export_dividend_daily_gbbq(gbbq_input, tdx_root, dividend_output, daily_output):
# 
# Path: kquant_data/processing/utils.py
# def filter_dataframe(df, index_name=None, start_date=None, end_date=None, fields=None):
#     if index_name is not None:
#         df['index_datetime'] = df[index_name].apply(yyyyMMddHHmm_2_datetime)
#         df = df.set_index('index_datetime')
#     # 过滤时间
#     if start_date is not None or end_date is not None:
#         df = df[start_date:end_date]
#     # 过滤字段
#     if fields is not None:
#         df = df[fields]
#     return df
# 
# def bar_convert(df, rule='1H'):
#     """
#     数据转换
#     :param df:
#     :param rule:
#     :return:
#     """
#     how_dict = {
#         'DateTime': 'first',
#         'Open': 'first',
#         'High': 'max',
#         'Low': 'min',
#         'Close': 'last',
#         'Amount': 'sum',
#         'Volume': 'sum',
#         'backward_factor': 'last',
#         'forward_factor': 'last',
#     }
#     # how_dict = {
#     #     'Symbol': 'first',
#     #     'DateTime': 'first',
#     #     'TradingDay': 'first',
#     #     'ActionDay': 'first',
#     #     'Time': 'first',
#     #     'BarSize': 'first',
#     #     'Pad': 'min',
#     #     'Open': 'first',
#     #     'High': 'max',
#     #     'Low': 'min',
#     #     'Close': 'last',
#     #     'Volume': 'sum',
#     #     'Amount': 'sum',
#     #     'OpenInterest': 'last',
#     #     'Settle': 'last',
#     #     'AdjustFactorPM': 'last',
#     #     'AdjustFactorTD': 'last',
#     #     'BAdjustFactorPM': 'last',
#     #     'BAdjustFactorTD': 'last',
#     #     'FAdjustFactorPM': 'last',
#     #     'FAdjustFactorTD': 'last',
#     #     'MoneyFlow': 'sum',
#     # }
#     columns = df.columns
#     new = df.resample(rule, closed='left', label='left').apply(how_dict)
# 
#     new.dropna(inplace=True)
#     # 有些才上市没多久的，居然开头有很多天为空白，需要删除,如sh603990
#     new = new[new['Open'] != 0]
# 
#     # 居然位置要调整一下
#     new = new[columns]
# 
#     # 由于存盘时是用的DateTime这个字段，不是index上的时间，这会导致其它软件读取数据时出错，需要修正数据
#     new['DateTime'] = new.index.map(datetime_2_yyyyMMddHHmm)
# 
#     return new
# 
# def multiprocessing_convert(multi, rule, _input, output, instruments, func_convert):
#     tic()
# 
#     if multi:
#         pool_size = multiprocessing.cpu_count() - 1
#         pool = multiprocessing.Pool(processes=pool_size)
#         func = partial(func_convert, rule, _input, output, instruments)
#         pool_outputs = pool.map(func, range(len(instruments)))
#         print('Pool:', pool_outputs)
#     else:
#         for i in range(len(instruments)):
#             func_convert(rule, _input, output, instruments, i)
# 
#     toc()
# 
# Path: kquant_data/stock/symbol.py
# def get_folder_symbols(folder, sub_folder):
#     path = os.path.join(folder, sub_folder, 'sh')
#     df_sh = get_symbols_from_path(path, "SSE")
#     path = os.path.join(folder, sub_folder, 'sz')
#     df_sz = get_symbols_from_path(path, "SZSE")
#     df = pd.concat([df_sh, df_sz])
# 
#     return df


import os
import pandas as pd
from kquant_data.config import __CONFIG_H5_STK_DIR__
from kquant_data.stock.stock import bars_to_h5
from kquant_data.processing.utils import filter_dataframe, bar_convert, multiprocessing_convert
from kquant_data.stock.symbol import get_folder_symbols
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
演示将5分钟数据转成1小时数据
"""



def _export_data(rule, _input, output, instruments, i):
    t = instruments.iloc[i]
    print("%d %s" % (i, t['local_symbol']))
    path = os.path.join(__CONFIG_H5_STK_DIR__, _input, t['market'], "%s.h5" % t['local_symbol'])
    df = None
    try:
        df = pd.read_hdf(path)
    except:
        pass
    if df is None:
        return None
    df = filter_dataframe(df, 'DateTime', None, None, None)
    df1 = bar_convert(df, rule)

# Path: kquant_data/config.py
# __CONFIG_H5_STK_DIR__ = r'D:\DATA_STK'
# 
# Path: kquant_data/stock/stock.py
# def sort_dividend(divs):
# def factor(daily, divs, ndigits):
# def adjust(df, adjust_type=None):
# def merge_adjust_factor(df, div):
# def read_h5_tdx(market, code, bar_size, h5_path, tdx_path, div_path):
# def _export_dividend_from_data(tdx_root, dividend_output, daily_output, data):
# def export_dividend_daily_dzh(dzh_input, tdx_root, dividend_output, daily_output):
# def export_dividend_daily_gbbq(gbbq_input, tdx_root, dividend_output, daily_output):
# 
# Path: kquant_data/processing/utils.py
# def filter_dataframe(df, index_name=None, start_date=None, end_date=None, fields=None):
#     if index_name is not None:
#         df['index_datetime'] = df[index_name].apply(yyyyMMddHHmm_2_datetime)
#         df = df.set_index('index_datetime')
#     # 过滤时间
#     if start_date is not None or end_date is not None:
#         df = df[start_date:end_date]
#     # 过滤字段
#     if fields is not None:
#         df = df[fields]
#     return df
# 
# def bar_convert(df, rule='1H'):
#     """
#     数据转换
#     :param df:
#     :param rule:
#     :return:
#     """
#     how_dict = {
#         'DateTime': 'first',
#         'Open': 'first',
#         'High': 'max',
#         'Low': 'min',
#         'Close': 'last',
#         'Amount': 'sum',
#         'Volume': 'sum',
#         'backward_factor': 'last',
#         'forward_factor': 'last',
#     }
#     # how_dict = {
#     #     'Symbol': 'first',
#     #     'DateTime': 'first',
#     #     'TradingDay': 'first',
#     #     'ActionDay': 'first',
#     #     'Time': 'first',
#     #     'BarSize': 'first',
#     #     'Pad': 'min',
#     #     'Open': 'first',
#     #     'High': 'max',
#     #     'Low': 'min',
#     #     'Close': 'last',
#     #     'Volume': 'sum',
#     #     'Amount': 'sum',
#     #     'OpenInterest': 'last',
#     #     'Settle': 'last',
#     #     'AdjustFactorPM': 'last',
#     #     'AdjustFactorTD': 'last',
#     #     'BAdjustFactorPM': 'last',
#     #     'BAdjustFactorTD': 'last',
#     #     'FAdjustFactorPM': 'last',
#     #     'FAdjustFactorTD': 'last',
#     #     'MoneyFlow': 'sum',
#     # }
#     columns = df.columns
#     new = df.resample(rule, closed='left', label='left').apply(how_dict)
# 
#     new.dropna(inplace=True)
#     # 有些才上市没多久的，居然开头有很多天为空白，需要删除,如sh603990
#     new = new[new['Open'] != 0]
# 
#     # 居然位置要调整一下
#     new = new[columns]
# 
#     # 由于存盘时是用的DateTime这个字段，不是index上的时间，这会导致其它软件读取数据时出错，需要修正数据
#     new['DateTime'] = new.index.map(datetime_2_yyyyMMddHHmm)
# 
#     return new
# 
# def multiprocessing_convert(multi, rule, _input, output, instruments, func_convert):
#     tic()
# 
#     if multi:
#         pool_size = multiprocessing.cpu_count() - 1
#         pool = multiprocessing.Pool(processes=pool_size)
#         func = partial(func_convert, rule, _input, output, instruments)
#         pool_outputs = pool.map(func, range(len(instruments)))
#         print('Pool:', pool_outputs)
#     else:
#         for i in range(len(instruments)):
#             func_convert(rule, _input, output, instruments, i)
# 
#     toc()
# 
# Path: kquant_data/stock/symbol.py
# def get_folder_symbols(folder, sub_folder):
#     path = os.path.join(folder, sub_folder, 'sh')
#     df_sh = get_symbols_from_path(path, "SSE")
#     path = os.path.join(folder, sub_folder, 'sz')
#     df_sz = get_symbols_from_path(path, "SZSE")
#     df = pd.concat([df_sh, df_sz])
# 
#     return df


# Path: demo_stock/C_1hour/A01_convert_5min_1hour.py
import os
import pandas as pd
from kquant_data.config import __CONFIG_H5_STK_DIR__
from kquant_data.stock.stock import bars_to_h5
from kquant_data.processing.utils import filter_dataframe, bar_convert, multiprocessing_convert
from kquant_data.stock.symbol import get_folder_symbols

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
演示将5分钟数据转成1小时数据
"""



def _export_data(rule, _input, output, instruments, i):
    t = instruments.iloc[i]
    print("%d %s" % (i, t['local_symbol']))
    path = os.path.join(__CONFIG_H5_STK_DIR__, _input, t['market'], "%s.h5" % t['local_symbol'])
    df = None
    try:
        df = pd.read_hdf(path)
    except:
        pass
    if df is None:
        return None
    df = filter_dataframe(df, 'DateTime', None, None, None)
    df1 = bar_convert(df, rule)

    date_output = os.path.join(__CONFIG_H5_STK_DIR__, output, t['market'], "%s.h5" % t['local_symbol'])