# Path: kquant_data/processing/utils.py
# def split_into_group(arr, n):
#     out = [arr[i:i + n] for i in range(0, len(arr), n)]
#     return out
# 
# def filter_dataframe(df, index_name=None, start_date=None, end_date=None, fields=None):
#     if index_name is not None:
#         df['index_datetime'] = df[index_name].apply(yyyyMMddHHmm_2_datetime)
#         df = df.set_index('index_datetime')
#     # 过滤时间
#     if start_date is not None or end_date is not None:
#         df = df[start_date:end_date]
#     # 过滤字段
#     if fields is not None:
#         df = df[fields]
#     return df
# 
# Path: kquant_data/utils/xdatetime.py
# def tic():
#     """
#     对应MATLAB中的tic
#     :return:
#     """
#     globals()['tt'] = time.clock()
# 
# def toc():
#     """
#     对应MATLAB中的toc
#     :return:
#     """
#     t = time.clock() - globals()['tt']
#     print('\nElapsed time: %.8f seconds\n' % t)
#     return t
# 
# Path: kquant_data/xio/h5.py
# def read_h5(path, dateset_name):
#     """
#     将简单数据读取出来
#     返回的东西有头表，就是DataFrame，没表头就是array
#     :param path:
#     :param dateset_name:
#     :return:
#     """
#     f = h5py.File(path, 'r')
# 
#     d = f[dateset_name][:]
# 
#     f.close()
#     return d
# 
# Path: kquant_data/stock/symbol.py
# def get_symbols_from_path(path, exchange):
#     """
#     指定目录，将目录转成合约列表
#     :param path:
#     :param exchange:
#     :return:
#     """
#     list1 = []
#     list2 = []
#     list3 = []
#     list4 = []
#     list5 = []
#     for dirpath, dirnames, filenames in os.walk(path):
#         for filename in filenames:
#             list1.append(filename[:8])
#             list2.append(filename[:2])
#             list3.append(filename[2:8])
#             list4.append("%s.%s" % (filename[2:8], exchange))
#             list5.append("%s.%s" % (filename[2:8], filename[:2].upper()))
# 
#     df = pd.DataFrame({"local_symbol": list1, "market": list2, "code": list3, "symbol": list4, "wind_code": list5})
# 
#     return df


import gc
import multiprocessing
import os
import shutil
import h5py
import numpy as np
import pandas as pd
from functools import partial
from .utils import split_into_group, filter_dataframe
from ..utils.xdatetime import tic, toc
from ..xio.h5 import read_h5
from ..stock.symbol import get_symbols_from_path
先取交易日，再取合约列表，然后全加载分成几个大表
"""




class MergeBar(object):
    def __init__(self, folder):
        self.prefix = 'tmp'
        self.folder = folder
        self.datetime = None
        self.instruments = None
        self.instruments_group = None
        self.fields = None
        self.group_len = 300
        # datetime与bar_size是相关联的
        self.bar_size = 86400
        self.init_datetime()
        self.init_symbols()
        self.init_fields()

    def init_datetime(self):
        path = os.path.join(self.folder, 'DateTime.csv')
        self.datetime.to_csv(path)

    def init_symbols(self):
        # 不再从导出列表中取，而是从文件夹中推算
        path = os.path.join(self.folder, 'sh')
        df_sh = get_symbols_from_path(path, "SSE")
        path = os.path.join(self.folder, 'sz')

# Path: kquant_data/processing/utils.py
# def split_into_group(arr, n):
#     out = [arr[i:i + n] for i in range(0, len(arr), n)]
#     return out
# 
# def filter_dataframe(df, index_name=None, start_date=None, end_date=None, fields=None):
#     if index_name is not None:
#         df['index_datetime'] = df[index_name].apply(yyyyMMddHHmm_2_datetime)
#         df = df.set_index('index_datetime')
#     # 过滤时间
#     if start_date is not None or end_date is not None:
#         df = df[start_date:end_date]
#     # 过滤字段
#     if fields is not None:
#         df = df[fields]
#     return df
# 
# Path: kquant_data/utils/xdatetime.py
# def tic():
#     """
#     对应MATLAB中的tic
#     :return:
#     """
#     globals()['tt'] = time.clock()
# 
# def toc():
#     """
#     对应MATLAB中的toc
#     :return:
#     """
#     t = time.clock() - globals()['tt']
#     print('\nElapsed time: %.8f seconds\n' % t)
#     return t
# 
# Path: kquant_data/xio/h5.py
# def read_h5(path, dateset_name):
#     """
#     将简单数据读取出来
#     返回的东西有头表，就是DataFrame，没表头就是array
#     :param path:
#     :param dateset_name:
#     :return:
#     """
#     f = h5py.File(path, 'r')
# 
#     d = f[dateset_name][:]
# 
#     f.close()
#     return d
# 
# Path: kquant_data/stock/symbol.py
# def get_symbols_from_path(path, exchange):
#     """
#     指定目录，将目录转成合约列表
#     :param path:
#     :param exchange:
#     :return:
#     """
#     list1 = []
#     list2 = []
#     list3 = []
#     list4 = []
#     list5 = []
#     for dirpath, dirnames, filenames in os.walk(path):
#         for filename in filenames:
#             list1.append(filename[:8])
#             list2.append(filename[:2])
#             list3.append(filename[2:8])
#             list4.append("%s.%s" % (filename[2:8], exchange))
#             list5.append("%s.%s" % (filename[2:8], filename[:2].upper()))
# 
#     df = pd.DataFrame({"local_symbol": list1, "market": list2, "code": list3, "symbol": list4, "wind_code": list5})
# 
#     return df


# Path: kquant_data/processing/MergeBar.py
import gc
import multiprocessing
import os
import shutil
import h5py
import numpy as np
import pandas as pd
from functools import partial
from .utils import split_into_group, filter_dataframe
from ..utils.xdatetime import tic, toc
from ..xio.h5 import read_h5
from ..stock.symbol import get_symbols_from_path

先取交易日，再取合约列表，然后全加载分成几个大表
"""




class MergeBar(object):
    def __init__(self, folder):
        self.prefix = 'tmp'
        self.folder = folder
        self.datetime = None
        self.instruments = None
        self.instruments_group = None
        self.fields = None
        self.group_len = 300
        # datetime与bar_size是相关联的
        self.bar_size = 86400
        self.init_datetime()
        self.init_symbols()
        self.init_fields()

    def init_datetime(self):
        path = os.path.join(self.folder, 'DateTime.csv')
        self.datetime.to_csv(path)

    def init_symbols(self):
        # 不再从导出列表中取，而是从文件夹中推算
        path = os.path.join(self.folder, 'sh')
        df_sh = get_symbols_from_path(path, "SSE")
        path = os.path.join(self.folder, 'sz')

        df_sz = get_symbols_from_path(path, "SZSE")