# Path: kquant_data/wind/tdays.py
# def read_tdays(path):
#     try:
#         df = pd.read_csv(path, parse_dates=True)
#     except:
#         return None
# 
#     df['date'] = pd.to_datetime(df['date'])
#     df.index = df['date']
#     return df
# 
# Path: kquant_data/config.py
# __CONFIG_TDAYS_SHFE_FILE__ = os.path.join(__CONFIG_H5_FUT_DIR__, 'tdays', 'SHFE.csv')
# 
# Path: kquant_data/wind_resume/wset.py
# def resume_download_futureoir(w, trading_days, root_path, windcode, adjust_trading_days):
#     dir_path = os.path.join(root_path, windcode)
#     os.makedirs(dir_path, exist_ok=True)
# 
#     # 从第一个有数据的部分开始第一条,这个需要根据自己的情况进行处理
#     if adjust_trading_days:
#         for dirpath, dirnames, filenames in os.walk(dir_path):
#             for filename in filenames:
#                 trading_days = trading_days[filename[:10]:]
#                 break
# 
#     # 分成list套list，只要时间超长就分新的一组
#     last_date = pd.Timestamp(1900, 1, 1).tz_localize(None)
#     day_ranges = []
#     day_range = []
#     for day in trading_days['date']:
#         path = os.path.join(root_path, windcode, day.strftime('%Y-%m-%d.csv'))
#         if os.path.exists(path):
#             continue
#         else:
#             if (day - last_date).days > 20 or len(day_range) >= 30:
#                 day_ranges.append(day_range)
#                 day_range = []
#             last_date = day
#             day_range.append(day)
#     # 将最后一个添加进去
#     day_ranges.append(day_range)
# 
#     for day_range in day_ranges:
#         if len(day_range) > 0:
#             ret = download_futureoir_day_range(w, day_range, windcode, root_path)
#             if ret == 0:
#                 print("下载数据为空，有可能才上市没多久，需要再下载")
#                 break
# 
#     print('处理完毕！可能有部分数据由于超时没有下载成功，可再运行一次脚本')
# 
# Path: kquant_data/future/symbol.py
# def get_actvie_products_wind():
#     _lst = get_actvie_products()
#     __lst = [get_wind_code(x).upper() for x in _lst]
#     return __lst
# 
# def get_all_products_wind():
#     _lst = get_all_products()
#     __lst = [get_wind_code(x).upper() for x in _lst]
#     return __lst
# 
# Path: kquant_data/config.py
# __CONFIG_H5_FUT_DATA_DIR__ = os.path.join(__CONFIG_H5_FUT_DIR__, 'data')


from WindPy import w
from datetime import datetime,timedelta
from kquant_data.wind.tdays import read_tdays
from kquant_data.config import __CONFIG_TDAYS_SHFE_FILE__
from kquant_data.wind_resume.wset import resume_download_futureoir
from kquant_data.future.symbol import get_actvie_products_wind,get_all_products_wind
from kquant_data.config import __CONFIG_H5_FUT_DATA_DIR__
import os
import numpy as np
import pandas as pd
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
同一品种的下载流程
"""

if __name__ == '__main__':
    # 很多品种的2016-04-21没有数据，如TA.CZC\ZC.CZC\SR.CZC\OI.CZC\MA.CZC\RM.CZC\FG.CZC\CF.CZC\WH.CZC，都郑商所

    # w = None
    w.start()

    # 最新的一天由于没有数据，会导致参与下载，如果中间某一天缺失，会导致大量下载数据，需要排除
    date_str = (datetime.today() - timedelta(days=0)).strftime('%Y-%m-%d')
    # date_str = datetime.today().strftime('%Y-%m-%d')
    # IF一类的起始时间是从2010年开始，不能再从2009年开始遍历了

    trading_days = read_tdays(__CONFIG_TDAYS_SHFE_FILE__)
    # trading_days = trading_days['2016-01-01':date_str]
    trading_days = trading_days['2017-10-01':date_str]

    # 下载后存下
    windcodes = get_all_products_wind()

# Path: kquant_data/wind/tdays.py
# def read_tdays(path):
#     try:
#         df = pd.read_csv(path, parse_dates=True)
#     except:
#         return None
# 
#     df['date'] = pd.to_datetime(df['date'])
#     df.index = df['date']
#     return df
# 
# Path: kquant_data/config.py
# __CONFIG_TDAYS_SHFE_FILE__ = os.path.join(__CONFIG_H5_FUT_DIR__, 'tdays', 'SHFE.csv')
# 
# Path: kquant_data/wind_resume/wset.py
# def resume_download_futureoir(w, trading_days, root_path, windcode, adjust_trading_days):
#     dir_path = os.path.join(root_path, windcode)
#     os.makedirs(dir_path, exist_ok=True)
# 
#     # 从第一个有数据的部分开始第一条,这个需要根据自己的情况进行处理
#     if adjust_trading_days:
#         for dirpath, dirnames, filenames in os.walk(dir_path):
#             for filename in filenames:
#                 trading_days = trading_days[filename[:10]:]
#                 break
# 
#     # 分成list套list，只要时间超长就分新的一组
#     last_date = pd.Timestamp(1900, 1, 1).tz_localize(None)
#     day_ranges = []
#     day_range = []
#     for day in trading_days['date']:
#         path = os.path.join(root_path, windcode, day.strftime('%Y-%m-%d.csv'))
#         if os.path.exists(path):
#             continue
#         else:
#             if (day - last_date).days > 20 or len(day_range) >= 30:
#                 day_ranges.append(day_range)
#                 day_range = []
#             last_date = day
#             day_range.append(day)
#     # 将最后一个添加进去
#     day_ranges.append(day_range)
# 
#     for day_range in day_ranges:
#         if len(day_range) > 0:
#             ret = download_futureoir_day_range(w, day_range, windcode, root_path)
#             if ret == 0:
#                 print("下载数据为空，有可能才上市没多久，需要再下载")
#                 break
# 
#     print('处理完毕！可能有部分数据由于超时没有下载成功，可再运行一次脚本')
# 
# Path: kquant_data/future/symbol.py
# def get_actvie_products_wind():
#     _lst = get_actvie_products()
#     __lst = [get_wind_code(x).upper() for x in _lst]
#     return __lst
# 
# def get_all_products_wind():
#     _lst = get_all_products()
#     __lst = [get_wind_code(x).upper() for x in _lst]
#     return __lst
# 
# Path: kquant_data/config.py
# __CONFIG_H5_FUT_DATA_DIR__ = os.path.join(__CONFIG_H5_FUT_DIR__, 'data')


# Path: demo_future/A02_download_futureoir.py
from WindPy import w
from datetime import datetime,timedelta
from kquant_data.wind.tdays import read_tdays
from kquant_data.config import __CONFIG_TDAYS_SHFE_FILE__
from kquant_data.wind_resume.wset import resume_download_futureoir
from kquant_data.future.symbol import get_actvie_products_wind,get_all_products_wind
from kquant_data.config import __CONFIG_H5_FUT_DATA_DIR__
import os
import numpy as np
import pandas as pd

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
同一品种的下载流程
"""

if __name__ == '__main__':
    # 很多品种的2016-04-21没有数据，如TA.CZC\ZC.CZC\SR.CZC\OI.CZC\MA.CZC\RM.CZC\FG.CZC\CF.CZC\WH.CZC，都郑商所

    # w = None
    w.start()

    # 最新的一天由于没有数据，会导致参与下载，如果中间某一天缺失，会导致大量下载数据，需要排除
    date_str = (datetime.today() - timedelta(days=0)).strftime('%Y-%m-%d')
    # date_str = datetime.today().strftime('%Y-%m-%d')
    # IF一类的起始时间是从2010年开始，不能再从2009年开始遍历了

    trading_days = read_tdays(__CONFIG_TDAYS_SHFE_FILE__)
    # trading_days = trading_days['2016-01-01':date_str]
    trading_days = trading_days['2017-10-01':date_str]

    # 下载后存下
    windcodes = get_all_products_wind()

    root_path = os.path.join(__CONFIG_H5_FUT_DATA_DIR__, "futureoir")