<jupyter_start><jupyter_text>alpha1-alpha10因子risk—netural回测结果<jupyter_code>%matplotlib inline
import sys
sys.path.append('../../../')
sys.path.append('../')

from datetime import datetime, timedelta
from m1_xgb import *
from src.conf.configuration import regress_conf
import xgboost as xgb
import pandas as pd
from PyFin.api import *
from alphamind.api import *
from src.conf.models import *
import numpy as np
from alphamind.execution.naiveexecutor import NaiveExecutor
from matplotlib import pyplot as plt

data_source = 'postgresql+psycopg2://alpha:alpha@172.16.58.3:8889/alpha'
engine = SqlEngine(data_source)

universe = Universe('zz500')
freq = '2b'
benchmark_code = 905
start_date = '2010-01-01'
end_date = '2010-05-01'
ref_dates = makeSchedule(start_date, end_date, freq, 'china.sse')
horizon = map_freq(freq)
industry_name = 'sw'
industry_level = 1

factors_store = {
    'f0': CSQuantiles(LAST('alpha_1'), groups='sw1'), 'f1': CSQuantiles(LAST('alpha_2'), groups='sw1'), 'f2': CSQuantiles(LAST('alpha_3'), groups='sw1'),
    'f3': CSQuantiles(LAST('alpha_4'), groups='sw1'), 'f4': CSQuantiles(LAST('alpha_5'), groups='sw1'), 'f5': CSQuantiles(LAST('alpha_6'), groups='sw1'),
    'f6': CSQuantiles(LAST('alpha_7'), groups='sw1'), 'f7': CSQuantiles(LAST('alpha_8'), groups='sw1'), 'f8': CSQuantiles(LAST('alpha_9'), groups='sw1'),
    'f9': CSQuantiles(LAST('alpha_10'), groups='sw1'), 'f10': CSQuantiles(LAST('alpha_11'), groups='sw1'), 'f11': CSQuantiles(LAST('alpha_12'), groups='sw1'),
    'f12': CSQuantiles(LAST('alpha_13'), groups='sw1'), 'f13': CSQuantiles(LAST('alpha_14'), groups='sw1'), 'f14': CSQuantiles(LAST('alpha_15'), groups='sw1'),
    'f15': CSQuantiles(LAST('alpha_16'), groups='sw1'), 'f16': CSQuantiles(LAST('alpha_17'), groups='sw1'), 'f17': CSQuantiles(LAST('alpha_18'), groups='sw1'),
    'f18': CSQuantiles(LAST('alpha_19'), groups='sw1'), 'f19': CSQuantiles(LAST('alpha_20'), groups='sw1'), 'f20': CSQuantiles(LAST('alpha_21'), groups='sw1'),
    'f21': CSQuantiles(LAST('alpha_22'), groups='sw1'), 'f22': CSQuantiles(LAST('alpha_23'), groups='sw1'), 'f23': CSQuantiles(LAST('alpha_24'), groups='sw1'),
    'f24': CSQuantiles(LAST('alpha_25'), groups='sw1'), 'f25': CSQuantiles(LAST('alpha_26'), groups='sw1'), 'f26': CSQuantiles(LAST('alpha_27'), groups='sw1'),
    'f27': CSQuantiles(LAST('alpha_28'), groups='sw1'), 'f28': CSQuantiles(LAST('alpha_29'), groups='sw1'), 'f29': CSQuantiles(LAST('alpha_30'), groups='sw1'),
    'f30': CSQuantiles(LAST('alpha_31'), groups='sw1'), 'f31': CSQuantiles(LAST('alpha_32'), groups='sw1'), 'f32': CSQuantiles(LAST('alpha_33'), groups='sw1'),
    'f33': CSQuantiles(LAST('alpha_34'), groups='sw1'), 'f34': CSQuantiles(LAST('alpha_35'), groups='sw1'), 'f35': CSQuantiles(LAST('alpha_36'), groups='sw1'),
    'f36': CSQuantiles(LAST('alpha_37'), groups='sw1'), 'f37': CSQuantiles(LAST('alpha_38'), groups='sw1'), 'f38': CSQuantiles(LAST('alpha_39'), groups='sw1'),
    'f39': CSQuantiles(LAST('alpha_40'), groups='sw1'), 'f40': CSQuantiles(LAST('alpha_41'), groups='sw1'), 'f41': CSQuantiles(LAST('alpha_42'), groups='sw1'),}

factor_data_org = engine.fetch_factor_range(universe, factors_store, dates=ref_dates, used_factor_tables=[Alpha191])


industry = engine.fetch_industry_range(universe, dates=ref_dates)
factor_data = pd.merge(factor_data_org, industry, on=['trade_date', 'code']).fillna(0.)
risk_total = engine.fetch_risk_model_range(universe, dates=ref_dates)[1]

return_data = engine.fetch_dx_return_range(universe, dates=ref_dates, horizon=horizon, offset=0,benchmark = benchmark_code)

benchmark_total = engine.fetch_benchmark_range(dates=ref_dates, benchmark=benchmark_code)
industry_total = engine.fetch_industry_matrix_range(universe, dates=ref_dates, category=industry_name, level=industry_level)

weight_gap = 0.01
industry_names = industry_list(industry_name, industry_level)
constraint_risk = ['EARNYILD', 'LIQUIDTY', 'GROWTH', 'SIZE', 'BETA', 'MOMENTUM'] + industry_names
total_risk_names = constraint_risk + ['benchmark', 'total']

b_type = []
l_val = []
u_val = []

previous_pos = pd.DataFrame()
rets = []
turn_overs = []
leverags = []
trade_dates = []

for name in total_risk_names:
        if name == 'benchmark':
            b_type.append(BoundaryType.RELATIVE)
            l_val.append(0.0)
            u_val.append(1.0)
        elif name == 'total':
            b_type.append(BoundaryType.ABSOLUTE)
            l_val.append(.0)
            u_val.append(.0)
        else:
            b_type.append(BoundaryType.ABSOLUTE)
            l_val.append(-0.005)
            u_val.append(0.005)

bounds = create_box_bounds(total_risk_names, b_type, l_val, u_val)

train_data = pd.merge(factor_data, return_data, on=['trade_date', 'code']).dropna()

features = ['f0', 'f1', 'f2', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9']
label = ['dx']

# take ref_dates[i] as an example
for i, date_inde in enumerate(ref_dates):
    alpha_logger.info('{0} is start'.format(date_inde))
    # Machine learning model
    trade_date_pre = date_inde - timedelta(days=1)
    # Train data
    train = train_data[train_data.trade_date <= trade_date_pre].dropna()
    if len(train) <= 0:
        continue
    x_train = train[features]
    y_train = train[label]
    alpha_logger.info('len_x_train: {0}, len_y_train: {1}'.format(len(x_train.values), len(y_train.values)))
    alpha_logger.info('X_train.shape={0}, X_test.shape = {1}'.format(np.shape(x_train), np.shape(y_train)))
    
    # Train
    # xgb_setting
    regress_conf.xgb_config_r()
    regress_conf.cv_folds = None
    regress_conf.early_stop_round = 4
    regress_conf.max_round = 10
    # Training
    tic = time.time()
    alpha_logger.info('X_train.shape={}, X_test.shape = {}'.format(np.shape(x_train), np.shape(y_train)))
    xgb_model = XGBooster(regress_conf)
    best_score, best_round, cv_rounds, best_model = xgb_model.fit(x_train, y_train)
    alpha_logger.info('Training time cost {}s'.format(time.time() - tic))
    alpha_logger.info('best_score = {}, best_round = {}'.format(best_score, best_round))
    
    # Test data
    # test_x = factor_data[factor_data.trade_date == date_inde]
    # test_y_excess = return_data[return_data.trade_date == date_inde]   
    # total_data_test_excess = pd.merge(test_x, test_y_excess, on=['trade_date', 'code']).dropna()
    total_data_test_excess = train_data[train_data.trade_date == date_inde]
    total_data_test_excess_pre = total_data_test_excess
    if len(total_data_test_excess) <= 0:
        continue

    industry_matrix = industry_total[industry_total.trade_date == date_inde]
    benchmark_w = benchmark_total[benchmark_total.trade_date == date_inde]
    risk_matrix = risk_total[risk_total.trade_date == date_inde]

    total_data = pd.merge(industry_matrix, benchmark_w, on=['code'], how='left').fillna(0.)
    total_data = pd.merge(total_data, risk_matrix, on=['code'])
    alpha_logger.info('{0} len_total_data: {1}'.format(date_inde, len(total_data)))

    total_data_test_excess = pd.merge(total_data, total_data_test_excess, on=['code'])
    alpha_logger.info('{0} len_total_data_test_excess: {1}'.format(date_inde, len(total_data_test_excess)))
    
    if len(total_data_test_excess_pre) != len(total_data_test_excess):
        print('total_data_test_exces_pre: %s' % len(total_data_test_excess_pre))
        print('total_data_test_excess: %s' % len(total_data_test_excess))
        
    codes = total_data_test_excess.code.values.tolist()
    alpha_logger.info('{0} full re-balance: {1}'.format(date_inde, len(codes)))
    dx_returns = return_data[return_data.trade_date == date_inde][['code', 'dx']]

    benchmark_w = total_data_test_excess.weight.values
    alpha_logger.info('shape_of_benchmark_w: {}'.format(np.shape(benchmark_w)))
    is_in_benchmark = (benchmark_w > 0.).astype(float).reshape((-1, 1))
    total_risk_exp = np.concatenate([total_data_test_excess[constraint_risk].values.astype(float),
                                     is_in_benchmark,
                                     np.ones_like(is_in_benchmark)],
                                     axis=1)
    total_risk_exp = pd.DataFrame(total_risk_exp, columns=total_risk_names)
    alpha_logger.info('shape_of_total_risk_exp: {}'.format(np.shape(total_risk_exp)))
    constraints = LinearConstraints(bounds, total_risk_exp, benchmark_w)
    alpha_logger.info('shape_of_constraints: {}'.format(np.shape(constraints.risk_targets())))

    lbound = np.maximum(0., benchmark_w - weight_gap)
    ubound = weight_gap + benchmark_w
    
    # predict
    x_pred = total_data_test_excess[features]
    dpred = xgb.DMatrix(x_pred.values)
    predict_xgboost = best_model.predict(dpred)
    a = np.shape(predict_xgboost)
    predict_xgboost = np.reshape(predict_xgboost, (a[0], -1)).astype(np.float64)
    alpha_logger.info('shape_of_predict_xgboost: {}'.format(np.shape(predict_xgboost)))
    
    # backtest
    target_pos, _ = er_portfolio_analysis(predict_xgboost,
                                          total_data_test_excess['industry'].values,
                                          None,
                                          constraints,
                                          False,
                                          benchmark_w,
                                          method = 'risk_neutral',
                                          lbound=lbound,
                                          ubound=ubound)
    
    alpha_logger.info('target_pos: {}'.format(np.shape(target_pos)))
    target_pos['code'] = codes
    result = pd.merge(target_pos, dx_returns, on=['code'])
    ret = result.weight.values @ (np.exp(result.dx.values) - 1.)
    trade_dates.append(date_inde)
    rets.append(np.log(1. + ret))
    alpha_logger.info('{0} is finished'.format(date_inde))
    
    
print(len(rets), len(trade_dates))
ret_df = pd.DataFrame({'xgb_regress': rets}, index=trade_dates)
ret_df.loc[advanceDateByCalendar('china.sse', ref_dates[-1], freq).strftime('%Y-%m-%d')] = 0.
ret_df = ret_df.shift(1)
ret_df.iloc[0] = 0.

ret_df[['xgb_regress']].cumsum().plot(figsize=(12, 6), 
                                      title='Fixed freq rebalanced: {0}'.format(freq))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15881429132312525, 'fraction_numerical': 0.0298390263054574, 'mean_word_length': 4.301248699271592, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '112790', 'license': 'Apache-2.0', 'n_tokens_mistral': 3772, 'n_tokens_neox': 3592, 'n_words': 542, 'path': 'src/stacking/notebooks/cross_section/rolling_backtest/xgb_apmd_bk_3.1.ipynb', 'repo_name': 'STHSF/MultiFactors', 'size': '10190'}