<jupyter_start><jupyter_text>data merging---------------------<jupyter_code>index_components_name = '500Weight'
benchmark = 'zz500'
return_data_500['dret'] = return_data_500['D1LogReturn']
index_data['ret'] = index_data[benchmark] / index_data[benchmark].shift(1) - 1.
index_data['dret_b'] = index_data['ret'] .shift(-2)
%%time
total_data = pd.merge(factor_data, prod_500[prod_factor_cols.append(pd.Series(['Date', 'Code']))], on=['Date', 'Code'])
total_data = pd.merge(total_data, common_500[common_500_factor_cols.append(pd.Series(['Date', 'Code']))], on=['Date', 'Code'])
total_data = pd.merge(total_data, index_data[['Date', 'dret_b']], on='Date', how='left')
total_data.dropna(inplace=True)
total_data = pd.merge(total_data, index_components[['Date', 'Code', index_components_name]], on=['Date', 'Code'], how='left')
total_data.fillna(0, inplace=True)
total_data = pd.merge(total_data, risk_factor_500, on=['Date', 'Code'])
total_data = pd.merge(total_data, return_data_500[['Date', 'Code', 'dret']], on=['Date', 'Code'])
total_data = total_data[total_data[index_components_name] != 0]
len(total_data)<jupyter_output><empty_output><jupyter_text>data processing---------------------------<jupyter_code>from alphamind.data.standardize import standardize
from alphamind.data.neutralize import neutralize
from alphamind.data.winsorize import winsorize_normal
total_factors = common_factor_cols.append(prod_factor_cols)
total_factors = total_factors.append(common_500_factor_cols)

#risk_factor_cols = risk_factor_cols[risk_factor_cols != 'Size']

all_factors = total_data[total_factors]
risk_factors = total_data[risk_factor_cols]
groups = total_data.Date.values.astype(np.int)
%%time
factor_processed = neutralize(risk_factors.values,
                              winsorize_normal(all_factors.values, groups=groups),
                              groups=groups)
normed_factor = pd.DataFrame(factor_processed, columns=total_factors, index=total_data.Date)<jupyter_output><empty_output><jupyter_text>Factor Performance (Long top)---------------------------------<jupyter_code>from alphamind.portfolio.rankbuilder import rank_build
%%time
use_rank = 100
pos_data = rank_build(normed_factor.values, use_rank, groups)
pos_df = pd.DataFrame(pos_data, columns=normed_factor.columns, index=normed_factor.index) / use_rank
ret_mat = (pos_df.values - total_data[[index_components_name]].values / 100.) * total_data[['dret']].values
ret_df = pd.DataFrame(ret_mat, columns=normed_factor.columns, index=normed_factor.index)
top_factors = ret_df.groupby(level=0).sum()[-90:].sum().abs().sort_values(ascending=False)[:10].index
ret_df[top_factors].groupby(level=0).sum()[-180:].cumsum().plot(figsize=(16, 8))
ret_df[top_factors].groupby(level=0).sum()[-90:].cumsum().plot(figsize=(16, 8))
ret_df[top_factors].groupby(level=0).sum()[-60:].cumsum().plot(figsize=(16, 8))
ret_df[top_factors].groupby(level=0).sum().cumsum().plot(figsize=(16, 8))
ret_df[prod_factor_cols].groupby(level=0).sum().cumsum().plot(figsize=(16, 8))
ret_df[prod_factor_cols].groupby(level=0).sum()[-90:].cumsum().plot(figsize=(16, 8))
pos_corr = pos_df.corr()
pos_corr = pos_corr.loc[total_factors, total_factors]
turn_over_table = {}
pos_df['Code'] = total_data.Code.values
pos_df.reset_index(inplace=True)

for name in total_factors:
    pos_series = pos_df[['Date', 'Code', name]]
    pivot_position = pos_series.pivot_table(name, index='Date', columns='Code').fillna(0.)
    turn_over_series = pivot_position.diff().abs().sum(axis=1)
    turn_over_table[name] = turn_over_series.values
    
turn_over_table = pd.DataFrame(turn_over_table, index=pos_df.Date.unique())
turn_over_table = turn_over_table[total_factors]
turn_over_table
from pandas import ExcelWriter
writer = ExcelWriter('Summary_500_long_top_tmp.xlsx')
ret_series = ret_df.groupby(level=0).sum().dropna()
ret_series.to_excel(writer, 'ret_series')
pos_corr.to_excel(writer, 'pos_corr')
turn_over_table.to_excel(writer, 'turn_over')
writer.close()<jupyter_output><empty_output><jupyter_text>Factor Performance (risk neutral)---------------------------------<jupyter_code>from alphamind.portfolio.linearbuilder import linear_build
total_data[total_factors] = normed_factor.values
total_data
marke_netural_pos = {}
for i, name in enumerate(total_factors):

    lbound_exposure = -1e-2 * np.ones(len(risk_factor_cols))
    ubound_exposure = 1e-2 * np.ones(len(risk_factor_cols))

    def get_benchmark_match_pos(x, name):
        er = x[name].values
        bm = x[index_components_name].values / 100.
        lbound = 0.
        ubound = 0.01 + bm
        risk_exposure = x[risk_factor_cols].values

        status, value , ret = linear_build(er,
                                           lbound=lbound,
                                           ubound=ubound,
                                           risk_exposure=risk_exposure,
                                           bm=bm,
                                           risk_target=(lbound_exposure, ubound_exposure),
                                           solver='GLPK')

        if status != 'optimal':
            return pd.Series(np.ones(len(er)) / len(er))
        else:
            return pd.Series(ret)
    
    look_into = risk_factor_cols.append(pd.Series([index_components_name, 'Date', name]))
    res = total_data[look_into].groupby('Date').apply(get_benchmark_match_pos, name=name).values
    marke_netural_pos[name] = res
    print('{0}: Factor {1} is finished'.format(i, name))
pos_df = pd.DataFrame(marke_netural_pos, index=total_data.Date)
ret_mat = (pos_df.values - total_data[[index_components_name]].values / 100.) * total_data[['dret']].values
ret_df = pd.DataFrame(ret_mat, columns=pos_df.columns, index=normed_factor.index)
ret_df[prod_factor_cols].groupby(level=0).sum().cumsum().plot(figsize=(16, 8))
ret_df[prod_factor_cols].groupby(level=0).sum()[-90:].cumsum().plot(figsize=(16, 8))
pos_corr = pos_df.corr()
pos_corr.loc[prod_factor_cols.tolist(), prod_factor_cols.tolist()]
turn_over_table = {}
pos_df['Code'] = total_data.Code.values
pos_df.reset_index(inplace=True)

for name in total_factors:
    pos_series = pos_df[['Date', 'Code', name]]
    pivot_position = pos_series.pivot_table(name, index='Date', columns='Code').fillna(0.)
    turn_over_series = pivot_position.diff().abs().sum(axis=1)
    turn_over_table[name] = turn_over_series.values
    
turn_over_table = pd.DataFrame(turn_over_table, index=pos_df.Date.unique())
turn_over_table = turn_over_table[total_factors]
turn_over_table
pos_corr = pos_corr.loc[total_factors, total_factors]

writer = ExcelWriter('Summary_500_risk_neutral_tmp.xlsx')
ret_series = ret_df.groupby(level=0).sum().dropna()
ret_series.to_excel(writer, 'ret_series')
pos_corr.to_excel(writer, 'pos_corr')
turn_over_table.to_excel(writer, 'turn_over')
writer.close()
turn_over_table<jupyter_output><empty_output><jupyter_text>Raw Product Factor -----------------------------------------<jupyter_code>risk_factor_values = total_data[risk_factor_cols].values
index_components_values = total_data[[index_components_name]].values / 100.
for i, name in enumerate(total_factors):
    pos_values = pos_df[[name]].values
    risk_values = (pos_values - index_components_values) * risk_factor_values
    risk_tables = pd.DataFrame(risk_values, columns=risk_factor_cols, index=total_data.Date)
    aggregated_risk = risk_tables.groupby(level=0).sum()
    print('{0}: Factor {1}, {2}, {3}'.format(i, name, aggregated_risk.min(),aggregated_risk.max()))
aggregated_risk.max()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16083267037920976, 'fraction_numerical': 0.016706443914081145, 'mean_word_length': 4.802307692307692, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '588852', 'license': 'MIT', 'n_tokens_mistral': 2741, 'n_tokens_neox': 2566, 'n_words': 346, 'path': 'notebooks/zz500_factor_analysis.ipynb', 'repo_name': 'lion-sing/alpha-mind', 'size': '7542'}