<jupyter_start><jupyter_text>基础因子实时计算<jupyter_code>%matplotlib inline
import sys
sys.path.append('../')
sys.path.append('../../')
sys.path.append('../../../')
sys.path.append('../../../../')
sys.path.append('../../../../../')

import pandas as pd
import numpy as np
import seaborn as sns
from datetime import datetime
from matplotlib import pyplot as plt

from PyFin.api import *
from alphamind.api import *
from conf.models import *
from conf.config import*
from data.engines.model import Record
from alphamind.execution.naiveexecutor import NaiveExecutor
from stacking import factor_store, feature_list
from optimization.bayesopt.bayes_optimization_xgb import *
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
pd.set_option('max_colwidth', 100)<jupyter_output><empty_output><jupyter_text>因子数据准备<jupyter_code># 设置train和predict日期
start_date = '2019-10-24'  # trainning起始时间
end_date = '2020-01-09'    # predict时间, 调仓日

# 设置交易日
trade_end_date = '2020-01-10'        # 当前交易日期[predict的下一个交易日作为下单调仓日]

# 设置保存的文件目录
weekly = 'st'  # 测试文件目录
freq = '5b'

# 换手率控制
turn_over_rate = 2  # {s1:0.5, s2:0.3, s3:0.6, s4:0.8, s5:0.5}

# 组合优化参数
weight_gap = 0.02

universe = Universe('zz500')
benchmark_code = 905
horizon = map_freq(freq)
industry_name = 'sw'
industry_level = 1
# 
ref_dates = makeSchedule(start_date, end_date, freq, 'china.sse')

# 根据索引删除某些特殊日期{节假日, 零时停盘日等}
trash_date_list = [datetime(2020, 1, 1, 0, 0)]

def remove_date(ref_dates, trash_date_list):
    for i in trash_date_list:
        try:
            del(ref_dates[ref_dates.index(i)])
        except:
            continue
    return ref_dates

ref_dates = remove_date(ref_dates, trash_date_list)
                
# 前一个调仓日
ref_date_pre = ref_dates[-2]
# 当前调仓日
ref_date = ref_dates[-1]
print('前一个调仓日: %s' % ref_date_pre)
print('当前调仓日: %s' % ref_date)

# 获取前一个交易日, 用于获取前一个调仓日的持仓信息
date_range = makeSchedule(ref_date_pre, trade_end_date, '1d', 'china.sse')
date_range = remove_date(date_range, trash_date_list)
# 前一个交易日, 如果遇到上一个调仓日存在空缺如[2020.01.01], 需要特殊指定
# adjusted_date_pre = datetime(2019, 12, 26, 0, 0)
adjusted_date_pre = date_range[1]
# 当前交易日
adjusted_date = date_range[-1]

print('前一个交易日: %s' % adjusted_date_pre)
print('当前交易日: %s' % adjusted_date)
# 因子数据库
data_source = alpha_db
engine = SqlEngine(data_source)

# uqer因子列表
## CSQuantiles
basic_factor_store = factor_store.basic_factor_store
## LAST
# basic_factor_store = factor_store.basic_factor_last_store

# alpha191因子列表
alpha_factor_store = factor_store.alpha_factor_store
%%time
# 提取Uqer因子
basic_factor_org = engine.fetch_factor_range(universe, basic_factor_store, dates=ref_dates)
## 提取alpha191因子
# alpha191_factor_org = engine.fetch_factor_range(universe, 
#                                                 alpha_factor_store, 
#                                                 dates=ref_dates, 
#                                                 used_factor_tables=[Alpha191]).drop(['chgPct','secShortName'],
#                                                                                     axis=1)
## 合并所有的因子
# factor_data_org = pd.merge(basic_factor_org, alpha191_factor_org, on=['trade_date', 'code'], how='outer')
factor_data_org = basic_factor_org

assert len(set(factor_data_org['trade_date'])) == len(ref_dates)

# Plot factor value over time on code
from matplotlib import pyplot as plt
from pylab import rcParams
rcParams['figure.figsize'] = 10, 2 # width 10, height 8

ax = factor_data_org[factor_data_org['code'] == 6].plot(x='trade_date', y='ACCA', style='b-', grid=True)
ax.set_xlabel("date")
ax.set_ylabel("ACCA")
# 因子预处理
## 确失值填充
factor_mean = factor_data_org.mean()
factor_std = factor_data_org.std()
factor_data_org = factor_data_org.fillna(factor_mean)
%%time
# 获取所属行业
industry = engine.fetch_industry_range(universe, dates=ref_dates)
# factor_data = pd.merge(factor_data_org, industry, on=['trade_date', 'code']).fillna(0.)
factor_data = pd.merge(factor_data_org, industry, on=['trade_date', 'code'])

# 获取风险因子
risk_total = engine.fetch_risk_model_range(universe, dates=ref_dates)[1]

%%time
return_data = engine.fetch_dx_return_range(universe, dates=ref_dates, horizon=horizon, offset=0,benchmark = benchmark_code)
# return_data = engine.fetch_dx_return_range(universe, dates=ref_dates, horizon=horizon, offset=0)


%%time
benchmark_total = engine.fetch_benchmark_range(dates=ref_dates, benchmark=benchmark_code)
industry_total = engine.fetch_industry_matrix_range(universe, dates=ref_dates, category=industry_name, level=industry_level)

train_data = pd.merge(factor_data, return_data, on=['trade_date', 'code']).dropna()

# Plot adjusted close over time on code
from matplotlib import pyplot as plt
from pylab import rcParams
rcParams['figure.figsize'] = 10, 2 # width 10, height 8

ax = train_data[train_data['code'] == 6].plot(x='trade_date', y='dx', style='b-', grid=True)
ax.set_xlabel("date")
ax.set_ylabel("dx")
# 获取特征名
features = list(basic_factor_store.keys())
alpha_features = list(alpha_factor_store.keys())
# features = feature_list.uqer_features
# alpha_features = feature_list.alpha_features
# features.extend(alpha_features)

label = ['dx']<jupyter_output><empty_output><jupyter_text>模型训练<jupyter_code>from datetime import datetime, timedelta
from models.m1_xgb import *
from conf.configuration import xgb_conf
from data.engines.model import Record
import xgboost as xgb
import gc

alpha_logger.info('{0} is start'.format(ref_date))

# machine learning model
## Filter Training data
## 训练集构造
trade_date_pre = ref_date - timedelta(days=1)
# trade_date_pre_80 = ref_date - timedelta(days=80)

## 1、选择调仓日当天之前(不含当天)并且在80天以内的因子数据作为训练集.
# train = train_data[(train_data.trade_date <= trade_date_pre) & (trade_date_pre_80 <= train_data.trade_date)].dropna()
## 2、选择调仓日当天之前(不含当天)的因子数据作为训练集.
train = train_data[train_data.trade_date <= trade_date_pre].dropna()
alpha_logger.info('trade_date_pre {0}'.format(trade_date_pre))

if len(train) <= 0:
    alpha_logger.info('{0} HAS NO TRAIN DATA!!!'.format(ref_date))

x_train = train[features]
y_train = train[label]
alpha_logger.info('len_x_train: {0}, len_y_train: {1}'.format(len(x_train.values), len(y_train.values)))
alpha_logger.info('X_train.shape={0}, X_test.shape = {1}'.format(np.shape(x_train), np.shape(y_train)))<jupyter_output>2020-02-20 13:03:32,590 - ALPHA_MIND - INFO - 2020-01-09 00:00:00 is start
2020-02-20 13:03:32,629 - ALPHA_MIND - INFO - trade_date_pre 2020-01-08 00:00:00
2020-02-20 13:03:32,636 - ALPHA_MIND - INFO - len_x_train: 5500, len_y_train: 5500
2020-02-20 13:03:32,636 - ALPHA_MIND - INFO - X_train.shape=(5500, 423), X_test.shape = (5500, 1)<jupyter_text>超参寻优<jupyter_code>%%time
# load xgboost regression configuration
xgb_conf.xgb_config_r()
xgb_conf.cv_folds = None
xgb_conf.early_stop_round = 100
xgb_conf.max_round = 800
xgb_conf.params.update({'nthread': 12})

GPU_device = False
if GPU_device:
    # use GPUs
    xgb_conf.params.update({'tree_method': 'gpu_hist'})
alpha_logger.info("params before: \n{}".format(xgb_conf.params))
tic = time.time()

# hyper_parameters optimization
# opt_parameters = {'max_depth': (2, 12),
#                   'gamma': (0.001, 10.0),
#                   'min_child_weight': (0, 20),
#                   'max_delta_step': (0, 10),
#                   'subsample': (0.01, 0.99),
#                   'colsample_bytree': (0.01, 0.99)
#                  }

# opt_xgb = BayesOptimizationXGB('regression', x_train, y_train)
# params_op = opt_xgb.train_opt(opt_parameters)
# xgb_conf.params.update(params_op)
alpha_logger.info("params after: \n{}".format(xgb_conf.params))
alpha_logger.info("hyper params optimize time : {}".format(time.time() - tic))<jupyter_output>2020-02-20 13:03:32,642 - ALPHA_MIND - INFO - params before: 
{'booster': 'gbtree', 'objective': 'reg:linear', 'eval_metric': ['rmse', 'logloss'], 'nthread': 12, 'silent': 1, 'learning_rate': 0.01, 'max_depth': 5, 'eta': 0.03, 'alpha': 0, 'lambda': 0, 'subsample': 0.7, 'colsample_bytree': 0.5, 'min_child_weight': 3, 'seed': 2019}
2020-02-20 13:03:32,643 - ALPHA_MIND - INFO - params after: 
{'booster': 'gbtree', 'objective': 'reg:linear', 'eval_metric': ['rmse', 'logloss'], 'nthread': 12, 'silent': 1, 'learning_rate': 0.01, 'max_depth': 5, 'eta': 0.03, 'alpha': 0, 'lambda': 0, 'subsample': 0.7, 'colsample_bytree': 0.5, 'min_child_weight': 3, 'seed': 2019}
2020-02-20 13:03:32,644 - ALPHA_MIND - INFO - hyper params optimize time : 0.0004761219024658203<jupyter_text>训练<jupyter_code>%%time
# model training
xgb_model = XGBooster(xgb_conf)
alpha_logger.info('xgb_model params: \n{0}'.format(xgb_model.get_params()))

best_score, best_round, best_model = xgb_model.fit(x_train, y_train)
alpha_logger.info('Training time cost {}s'.format(time.time() - tic))
alpha_logger.info('best_score = {}, best_round = {}'.format(best_score, best_round))<jupyter_output>2020-02-20 13:03:32,649 - ALPHA_MIND - INFO - xgb_model params: 
{'booster': 'gbtree', 'objective': 'reg:linear', 'eval_metric': ['rmse', 'logloss'], 'nthread': 12, 'silent': 1, 'learning_rate': 0.01, 'max_depth': 5, 'eta': 0.03, 'alpha': 0, 'lambda': 0, 'subsample': 0.7, 'colsample_bytree': 0.5, 'min_child_weight': 3, 'seed': 2019}
2020-02-20 13:03:32,650 - ../../../../models/m1_xgb.py[line:95] - INFO: NonCrossValidation。。。。<jupyter_text>当天数据预测<jupyter_code># 取调仓日当天的因子数据作为输入.
# total_data_test_excess = train_data[train_data.trade_date == str(ref_date)]
total_data_test_excess = factor_data[factor_data.trade_date == ref_date]

if len(total_data_test_excess) <=0:
    alpha_logger.info('{} HAS NO DATA!!!'.format(ref_date))
    sys.exit()

alpha_logger.info('{0} total_data_test_excess: {1}'.format(ref_date, len(total_data_test_excess)))

# 获取调仓日当天的行业, 风险模型和基准权重数据
industry_matrix = industry_total[industry_total.trade_date == ref_date]
benchmark_weight = benchmark_total[benchmark_total.trade_date == ref_date]
risk_matrix = risk_total[risk_total.trade_date == ref_date]

total_data = pd.merge(industry_matrix, benchmark_weight, on=['code'], how='left').fillna(0.)
total_data = pd.merge(total_data, risk_matrix, on=['code'])
alpha_logger.info('{0} type_of_total_data: {1}'.format(ref_date, type(total_data)))
alpha_logger.info('{0} shape_of_total_data: {1}'.format(ref_date, np.shape(total_data)))
    
total_data_test_excess = pd.merge(total_data, total_data_test_excess, on=['code'])
alpha_logger.info('{0} len_of_total_data_test_excess: {1}'.format(ref_date, len(total_data_test_excess)))

# 解析股票代码
codes = total_data_test_excess.code.values.tolist()
   
# predict
# alpha_logger.info('total_data_test_excess: \n{}'.format(total_data_test_excess[['weight', 'code', 'industry']]))
x_pred = total_data_test_excess[features]
predict_xgboost = xgb_model.predict(best_model, x_pred)
# alpha_logger.info('predict_xgboost: {}'.format(predict_xgboost))

a = np.shape(predict_xgboost)
predict_xgboost = np.reshape(predict_xgboost, (a[0], -1)).astype(np.float64)
alpha_logger.info('shape_of_predict_xgboost: {}'.format(np.shape(predict_xgboost)))

# 收益率预测结果    
predict_xgboost_df = pd.DataFrame({'xgb_pre': list(predict_xgboost.reshape(-1))})
predict_xgboost_df['trade_date'] = ref_date
predict_xgboost_df['code'] = codes
predict_xgboost_df['code'] = predict_xgboost_df['code'].apply(lambda x: "{:06d}".format(x) + '.XSHG'
                                                              if len(str(x))==6 and str(x)[0] in '6' 
                                                              else "{:06d}".format(x) + '.XSHE')<jupyter_output>2020-02-20 13:03:37,120 - ALPHA_MIND - INFO - 2020-01-09 00:00:00 total_data_test_excess: 500
2020-02-20 13:03:37,133 - ALPHA_MIND - INFO - 2020-01-09 00:00:00 type_of_total_data: <class 'pandas.core.frame.DataFrame'>
2020-02-20 13:03:37,133 - ALPHA_MIND - INFO - 2020-01-09 00:00:00 shape_of_total_data: (500, 74)
2020-02-20 13:03:37,141 - ALPHA_MIND - INFO - 2020-01-09 00:00:00 len_of_total_data_test_excess: 500
2020-02-20 13:03:37,154 - ALPHA_MIND - INFO - shape_of_predict_xgboost: (500, 1)<jupyter_text>获取昨持仓信息<jupyter_code>from data.engines.sqlengine import SQLEngine

# 获取上个交易日的当前持仓
record_engine = SQLEngine('sqlite:///./{}/real_tune_record_without_alpha.db'.format(weekly))

try:
    # pos_record = record_engine.fetch_record('pos_record')
    # previous_pos = pos_record[pos_record['trade_date'] == adjusted_date_pre]
    previous_pos = record_engine.fetch_record_meta(Record, adjusted_date_pre)

except Exception as e:
    alpha_logger.info('pos_record Exception:{0}'.format(e))
    previous_pos = pd.DataFrame({'trade_date':[], 'weight':[],'industry':[], 'er':[],'code':[]})

alpha_logger.info('previous_pos_data: {0}, pos_len: {1}'.format(adjusted_date_pre, len(previous_pos)))

# 股票过滤, 组合优化之前过滤掉(未完成)
## 9:00--9:25之间进行涨跌停股票的实时筛选

# 导入昨持仓并与股票池中所有股票合并, 
if len(previous_pos) <= 0:
    current_position = None
else:
    previous_pos = total_data_test_excess[['code']].merge(previous_pos, on=['code'], how='left',).fillna(0)
    current_position = previous_pos.weight.values
alpha_logger.info('previous_pos:\n {}'.format(previous_pos.head()))

# previous_pos = total_data_test_excess[['code']].merge(previous_pos, on=['code'], how='left').fillna(0)
# current_position = previous_pos.weight.values

# print(current_position.shape)
# print(total_data_test_excess.shape)
# print(previous_pos.shape)<jupyter_output>2020-02-20 13:03:37,352 - ALPHA_MIND - INFO - previous_pos_data: 2020-01-03 00:00:00, pos_len: 0
2020-02-20 13:03:37,354 - ALPHA_MIND - INFO - previous_pos:
 Empty DataFrame
Columns: [trade_date, adjust_date, weight, industry, er, code]
Index: []<jupyter_text>组合优化<jupyter_code># Constraintes settings
industry_names = industry_list(industry_name, industry_level)
constraint_risk = ['EARNYILD', 'LIQUIDTY', 'GROWTH', 'SIZE', 'SIZENL', 'BETA', 'MOMENTUM'] + industry_names
total_risk_names = constraint_risk + ['benchmark', 'total']

best_industry_names = ['电子','家用电器','食品饮料','医药生物','通信']

effective_industry_names = ['建筑材料','机械设备','家用电器','交通运输','化工',
                            '电气设备','信息服务','银行',
                            '建筑装饰','计算机','轻工制造','交运设备','信息设备',
                            '建筑建材','商业贸易','房地产','汽车',
                            '公用事业','保险','休闲服务','证券','多元金融']

invalid_industry_names = ['农林牧渔','采掘', '钢铁', '有色金属', '纺织服装', '商业贸易',
                          '综合', '国防军工','传媒']

b_type = []
l_val = []
u_val = []

for name in total_risk_names:
    if name == 'benchmark':
        b_type.append(BoundaryType.RELATIVE)
        l_val.append(0.0)
        u_val.append(1.0)
    elif name == 'total':
        b_type.append(BoundaryType.ABSOLUTE)
        l_val.append(-0.0)
        u_val.append(0.0)
    elif name == 'SIZE':
        b_type.append(BoundaryType.ABSOLUTE)
        l_val.append(-1.0)
        u_val.append(1.0)
    elif name == 'SIZENL':
        b_type.append(BoundaryType.ABSOLUTE)
        l_val.append(-1.0)
        u_val.append(1.0)
    elif name in industry_names:
        b_type.append(BoundaryType.RELATIVE)
        l_val.append(-0.05)
        u_val.append(0.05)
#     elif name in best_industry_names:
#         b_type.append(BoundaryType.RELATIVE)
#         l_val.append(0.0)
#         u_val.append(0.05)
#     elif name in effective_industry_names:
#         b_type.append(BoundaryType.RELATIVE)
#         l_val.append(0.0)
#         u_val.append(0.001)
#     elif name in invalid_industry_names:
#         b_type.append(BoundaryType.RELATIVE)
#         l_val.append(0.0)
#         u_val.append(1.0)
    else:
        b_type.append(BoundaryType.ABSOLUTE)
        l_val.append(-2.0)
        u_val.append(2.0)
bounds = create_box_bounds(total_risk_names, b_type, l_val, u_val)

bounds['农林牧渔'].lower.val
%%time
benchmark_w = total_data_test_excess.weight.values
alpha_logger.info('type_of_benchmark_w: {}, shape_of_benchmark_w: {}'.format(type(benchmark_w), 
                                                                             np.shape(benchmark_w)))
is_in_benchmark = (benchmark_w > 0.).astype(float).reshape((-1, 1))

# 风险模型数据合并
total_risk_exp = np.concatenate([total_data_test_excess[constraint_risk].values.astype(float),
                                 is_in_benchmark,
                                 np.ones_like(is_in_benchmark)],
                                axis=1)

alpha_logger.info('shape_of_total_risk_exp_pre: {}'.format(np.shape(total_risk_exp)))
total_risk_exp = pd.DataFrame(total_risk_exp, columns=total_risk_names)
alpha_logger.info('shape_of_total_risk_exp: {}'.format(np.shape(total_risk_exp)))

constraints = LinearConstraints(bounds, total_risk_exp, benchmark_w)
alpha_logger.info('constraints: {0} in {1}'.format(np.shape(constraints.risk_targets()), ref_date))

lbound = np.maximum(0., benchmark_w - weight_gap)
ubound = weight_gap + benchmark_w
alpha_logger.info('lbound: {0} in {1}'.format(np.shape(lbound), ref_date))
alpha_logger.info('ubound: {0} in {1}'.format(np.shape(ubound), ref_date))

# 组合优化
executor = NaiveExecutor()
current_pos = pd.DataFrame()

try:
    target_pos, _ = er_portfolio_analysis(predict_xgboost, 
                                      total_data_test_excess['industry'].values,
                                      None,
                                      constraints,
                                      False,
                                      benchmark_w,
                                      method='risk_neutral',
                                      lbound=lbound,
                                      ubound=ubound,
                                      turn_over_target=turn_over_rate,
                                      current_position=current_position)
except:
    target_pos, _ = er_portfolio_analysis(predict_xgboost, 
                                      total_data_test_excess['industry'].values,
                                      None,
                                      constraints,
                                      False,
                                      benchmark_w,
                                      method='risk_neutral',
                                      lbound=lbound,
                                      ubound=ubound,
                                      current_position=current_position)
                  
alpha_logger.info('shape_of_target_pos: {}'.format(np.shape(target_pos)))
alpha_logger.info('len_codes:{}'.format(np.shape(codes)))
target_pos['code'] = codes
# alpha_logger.info('target_pos: \n{}'.format(target_pos))

# 换手率计算
executor.set_current(previous_pos)
turn_over_org, current_pos = executor.execute(target_pos=target_pos)
alpha_logger.info('turn_over_org: {}'.format(turn_over_org))
turn_over = turn_over_org / sum(target_pos.weight.values)
alpha_logger.info('turn_over: {}'.format(turn_over))

# 优化后仓位信息
## 调仓日
current_pos['adjust_date'] = ref_date
## 交易日
current_pos['trade_date'] = adjusted_date
alpha_logger.info('{} is finished'.format(adjusted_date))

# 取TOP N作为真实的下单股票
real_pos = current_pos.sort_values(by='weight', ascending=False)[:50]
real_pos['weight'] = real_pos['weight'] / real_pos['weight'].sum()
real_pos
# 保存当前持仓信息   
## 字段说明:{weight: 持股权重, industry:所属行业, er:收益率, code:股票代码, adjust_date:调仓日, trade_date:交易日}
## 注, adjust_date表示模型运行所需要的数据截止日, trade_date表示策略下单交易的日期
previous_record = record_engine.fetch_record_meta(Record, adjusted_date)
if len(previous_record) == 0:
    record_engine.write_data('pos_record', real_pos)
else:
    record_engine.del_historical_data(Record, adjusted_date)  # 删除同日期的历史数据
    tmp_record = record_engine.fetch_record_meta(Record, adjusted_date)
    if len(tmp_record) == 0:  # 删除成功
        record_engine.write_data('pos_record', real_pos)
    else:
        print('{} 的数据没有删除: {}'.format(adjusted_date, len(previous_record)))

# 生成交易记录
## 修改code格式
real_pos['code'] = real_pos['code'].apply(lambda x: "{:06d}".format(x) + '.SH' 
                                          if len(str(x))==6 and str(x)[0] in '6' 
                                          else "{:06d}".format(x) + '.SZ')

real_pos = real_pos.loc[:, ['code', 'weight', 'trade_date']]
real_pos.rename(columns={"code": "证券代码", "weight": "持仓权重", "trade_date": "成分日期"}, inplace=True)
real_pos['交易价格'] = 0

real_pos = real_pos[['证券代码', '持仓权重', '交易价格', '成分日期']].copy()
# real_pos.to_csv('./{}/base{}.csv'.format(weekly, str(adjusted_date.date())), encoding='utf_8_sig', index=False)
# 查看持仓记录
previous_record = record_engine.fetch_record_meta(Record, adjusted_date)
# previous_record = record_engine.fetch_record('pos_record')
# previous_record = previous_record[previous_record['trade_date'] == adjusted_date]
previous_record<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14474325246029426, 'fraction_numerical': 0.037367241547305856, 'mean_word_length': 3.887380952380952, 'pattern_counts': {'":': 3, '<': 35, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 41, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '73678', 'license': 'Apache-2.0', 'n_tokens_mistral': 8387, 'n_tokens_neox': 8028, 'n_words': 1174, 'path': 'src/stacking/notebooks/cross_section/RealOperation/run_realtime_analysis_test.ipynb', 'repo_name': 'STHSF/MultiFactors', 'size': '20526'}