<jupyter_start><jupyter_text>extract all monthly market_return (Make it fast)<jupyter_code>start = '20010101'
total_mr = []
mr_date = []
for i in tqdm.tqdm(range(246)):
    start = last_market_date(start)
    end = last_market_date(find_next_market_date(start))
    tobe = get_market_return_for_period(start, end)
    mr_date.append(end)
    total_mr.append(tobe)
    start = find_next_market_date(start)
def get_market_return_for_period2(date_from, date_to):
    return total_mr[mr_date.index(date_to)]
def get_market_return_for_period(date_from, date_to):
    first = get_market_data_on_date(find_next_market_date(date_from, today_opt = True), print_error = False).loc[:, ['종목코드', '종가', '시가총액']].set_index('종목코드')
    df = pd.DataFrame()
    flag = find_next_market_date(date_from, today_opt = True)
    while flag != date_to:
        flag = find_next_market_date(flag)
        df = pd.concat([df, get_market_data_on_date(flag, print_error = False).loc[:,['종목코드', '등락률']].set_index('종목코드').rename({'등락률':flag}, axis=1)], axis=1)
        if flag == date_to:
            second = get_market_data_on_date(flag, print_error = False).loc[:, ['종목코드', '종가']].set_index('종목코드')
    df = df.applymap(lambda x: (100 + x)/100)
    for i in range(len(df.columns)):
        if i == 0:
            geo_sum = df.iloc[:,i].copy()
        else:
            geo_sum *= df.values[:,i]
    compute_by_price = second['종가'] / first['종가']
    # 가정 : 3% 이상 차이가 날 수 없다. 계산상 최대 1프로 정도 차이지만, 좀 갭을 두었음.
    final_return = (compute_by_price - geo_sum < 0.03) * compute_by_price + (compute_by_price - geo_sum >= 0.03) * geo_sum
    final_return = pd.concat([final_return.loc[first.index].fillna(0.01), first['시가총액']], axis=1).rename({0:'등락률'}, axis=1)
    final_return['등락률'] = final_return['등락률'] -1
    return final_return
def get_momentum(data, before, after):
    # before는 기준시점, after는 모멘텀 체크 시점.
    # 즉, 12-1 모멘텀 체크를 위해서는 before에 12, after에 1 을 입력하면 된다.
    return data.T.pct_change(before - after, fill_method = None).shift(after).iloc[before:,:].T
# get momentum
def get_momentum_for_period(date_from, date_to, term, before, after):
    flag = last_market_date(date_from)<jupyter_output><empty_output><jupyter_text>Pfo Construction<jupyter_code>def make_portfolio(mask, data, market_bp= 'All', breakpoint=[0.5]):
    # 여기서 앞에 들어가는 거는 마스크, 뒤에 들어가는 거는 실제값. 
    max_num = int(mask.max()[0]+1)
    new_df = pd.DataFrame(index = mask.index, columns = mask.columns)
    for i in range(len(mask.columns)):
        for j in range(max_num):
            get_index = list(filter(lambda x: x in data.index, mask.iloc[:,i][mask.iloc[:,i]==j].index))
            pfo_mask = get_mask(pd.DataFrame(data.loc[get_index].iloc[:,i]), get_market_category(pd.DataFrame(data.loc[get_index].iloc[:,i])), market_bp, breakpoint)
            new_df.iloc[:,i].loc[pfo_mask.index] = pfo_mask.values[:,0]
    return new_df

def serialize_pfo(mask_list: list):
    # 사전식 배열. 앞에서부터 사전식으로 serialize하여 뱉음
    serial_num = [x.max().max()+1 for x in mask_list]
    for i in reversed(range(len(serial_num)-1)):
        serial_num[i] *= serial_num[i+1]
    serial_num += [1]
    serial_num = serial_num[1:]
    serial_elem = [x * mask_list[i] for i, x in enumerate(serial_num)]
    for i in range(len(serial_elem)-1):
        serial_elem[0] += serial_elem[i+1]
    return serial_elem[0]<jupyter_output><empty_output><jupyter_text>Regression 해볼 수 있도록 만들기<jupyter_code>def get_market_return(mask, term = 'd'):
    term_conservative = pd.Timestamp(mask.columns[1]) - pd.Timestamp(mask.columns[0]) - pd.Timedelta(days = 7)
    if pd.Timestamp(mask.columns[-1]) + term_conservative >  pd.Timestamp('20210630'):
        date_to = pd.Timestamp('20210630')
    else:
        date_to = pd.Timestamp(last_market_date((pd.Timestamp(mask.columns[-1]) + term_conservative).strftime('%Y%m%d')))
    date_from = pd.Timestamp(mask.columns[0])
    factor_days = (date_to - date_from).days
    factor_months = (date_to.year - date_from.year)*12 + (date_to.month - date_from.month)
    
    date_point = date_from.strftime('%Y%m%d')
    # this is for storing the last date
    date_point_marked = None
    
    total_return = []
    date_list = []
    
    if term == 'd':
        flag = -1
        date_point = find_next_market_date(date_point)
        for _ in tqdm.tqdm(range(factor_days)):
            if pd.Timestamp(find_next_market_date(date_point)) > pd.Timestamp(mask.columns[flag+1]):
                flag+=1
                c_mask = mask.iloc[:,flag]
            first = get_market_data_on_date(date_point, print_error = False).loc[:,['종목코드', '종가', '시가총액', '등락률']]
            date_list.append(date_point)
            date_point = find_next_market_date(date_point)
            
            market_cap = first['시가총액'].sum()
            R = first['등락률'] * (first['시가총액'] / market_cap)
            total_return.append(R.fillna(0).sum())
            
        return pd.DataFrame(total_return, index = date_list)
    elif term == 'm':
        flag = -1
        for _ in tqdm.tqdm(range(factor_months)):
            if flag +1 == mask.shape[1]:
                c_mask = mask.iloc[:,flag]
                date_point = find_next_market_date(date_point)
                date_point_marked = date_point
            elif pd.Timestamp(find_next_market_date(date_point)) > pd.Timestamp(mask.columns[flag+1]):
                flag+=1
                c_mask = mask.iloc[:,flag]
                date_point_marked = last_market_date(date_point)
            else:
                date_point_marked = date_point
            
            date_point = last_market_date(find_next_market_date(date_point_marked))
            return_value = get_market_return_for_period(date_point_marked, date_point)
            date_list.append(date_point[:6])
            market_cap = return_value['시가총액'].sum()
            R = return_value['등락률'] * (return_value['시가총액'] / market_cap)
            
            total_return.append(R.replace([np.inf, -np.inf], np.nan).fillna(0).sum())
        return pd.DataFrame(total_return, index = date_list)
# risk free rate
os.chdir('C:\\Users\\lyoo9\\_Moggle-Labs')
R_f = pd.read_excel('risk_free_rate.xls')
cd_91 = R_f.iloc[[1,6],1:]
cd_91 =cd_91.T.set_index(1)
cd_91.index = cd_91.index.map(lambda x:x[:-1])
cd_91.columns = ['cd_91']
os.chdir('C:\\Users\\lyoo9\\_System Trading\\Backend\\KRX_price')
# pfo가 주어지면, 이제 해당 column별로 수익률을 회귀해야 함.
# 기간은 201307~202106으로 하자.
R_p = pfo.iloc[6:,0] - cd_91[:-4]
R_m = R_m.iloc[6:,0] - cd_91[:-4]
equity_factor = (equity_factor.iloc[:,0] - equity_factor.iloc[:,1]).iloc[6:0]

df = pd.DataFrame([R_p, R_m, equity_factor])<jupyter_output><empty_output><jupyter_text>모듈화하기<jupyter_code>def get_risk_free_rate(pfo, risk_free):
    return risk_free.loc[pfo.index]

def get_ols_model(mask, R_m, pfo_column, risk_free, *args):
    R_f = (get_risk_free_rate(pfo_column, risk_free).astype(float)/12).squeeze()
    df = pd.DataFrame([(pfo_column - R_f.T).values.squeeze(), (R_m - R_f).values.squeeze()] + [x.values.squeeze() for x in args]).T
    df.columns = ['pfo', 'market'] + ['factor' + str(i) for i, x in enumerate(args)]
    var_str = ''
    for i in range(len(df.columns)-2):
        var_str += ' + ' + ['factor' + str(i) for i, x in enumerate(args)][i]
    model = ols(formula = 'pfo ~ market' + var_str,data = df).fit()
    return model<jupyter_output><empty_output><jupyter_text>이제 FF5 구현해보자.<jupyter_code>Market_cap = get_market_cap_for_period('20101231', '20191231')

equity = get_item_for_period('ifrs_Equity', '20101231', '20191231')

BEME = equity/Market_cap[0]
Market_cap = get_market_cap_for_period('20110630', '20200630')
# SMB
SMB_mask = get_mask(Market_cap[0], Market_cap[1], market_bp = 'KOSPI')

SMB = get_factor_on_date_by_mask(SMB_mask, term = 'm', winsorize_limits = 0.01, weight = 'VW')
SMB = get_factor_on_date_by_mask(SMB_mask, term = 'm', winsorize_limits = 0.01, weight = 'VW')
SMB2 = get_factor_on_date_by_mask2(SMB_mask, term = 'm', winsorize_limits = 0.01, weight = 'VW')
# HML
BEME_shifted = shift_date_quarter(BEME, 2)

HML_mask = get_mask(BEME_shifted, get_market_category(BEME_shifted), market_bp = 'KOSPI', breakpoint = [0.3, 0.7])

HML = get_factor_on_date_by_mask(HML_mask, term = 'm', winsorize_limits = 0.01, weight = 'VW')
# RMW 계산하기
# (영업이익 - 이자비용) / 총자산
Total_asset = get_item_for_period('ifrs_Assets', '20101231', '20191231')
fcost1 = get_item_for_period('ifrs_FinanceCosts', '20101231', '20191231')
fcost2 = get_item_for_period('dart_InterestExpenseFinanceExpense', '20101231', '20191231')
fcost = pd.concat([fcost1, fcost2], axis = 1).groupby(level=0, axis=1).last()
OI = get_item_for_period('dart_OperatingIncomeLoss', '20101231', '20191231')
RMW_base = (OI - fcost) / Total_asset

RMW_base_shifted = shift_date_quarter(RMW_base, 2)

RMW_mask = get_mask(RMW_base_shifted, get_market_category(RMW_base_shifted), market_bp = 'KOSPI', breakpoint = [0.3, 0.7])

RMW = get_factor_on_date_by_mask(RMW_mask, term = 'm', winsorize_limits = 0.01, weight = 'VW')
# CMA 계산하기

Total_asset = get_item_for_period('ifrs_Assets', '20091231', '20181231')
pnl = get_item_for_period('ifrs_ProfitLoss', '20101231', '20191231')
Total_asset_shifted = shift_date_quarter(Total_asset, 4)
CMA_base = pnl / Total_asset_shifted
CMA_base_shifted = shift_date_quarter(CMA_base, 2)
CMA_mask = get_mask(CMA_base_shifted, get_market_category(CMA_base_shifted), market_bp = 'KOSPI', breakpoint = [0.3, 0.7])
CMA = get_factor_on_date_by_mask(CMA_mask, term = 'm', winsorize_limits = 0.01, weight = 'VW')
pfo_return
# HML pfo
hml_pfo_mask = make_portfolio(get_mask(Market_cap[0], Market_cap[1], market_bp = 'KOSPI', breakpoint = [0.2, 0.4, 0.6, 0.8]), BEME_shifted, breakpoint = [0.2, 0.4, 0.6, 0.8])

pfo25 = serialize_pfo([get_mask(Market_cap[0], Market_cap[1], market_bp = 'KOSPI', breakpoint = [0.2, 0.4, 0.6, 0.8]), hml_pfo_mask])

pfo_return = get_factor_on_date_by_mask(pfo25, term = 'm', winsorize_limits=0.01, weight = 'VW')
pfo_return
coef = []
tvalues = []
r = []
p = []
# R_m = get_market_return(pfo25, term = 'm').squeeze()
for i in range(25):
    model = get_ols_model(pfo25, R_m, pfo_return.iloc[:,i], cd_91, SMB.iloc[:,0] - SMB.iloc[:,1], HML.iloc[:,2] - HML.iloc[:,0], RMW.iloc[:,2] - RMW.iloc[:,0], CMA.iloc[:,0] - CMA.iloc[:,2])
    coef.append(model.params)
    tvalues.append(model.tvalues)
    r.append(model.rsquared)
    p.append(model.pvalues)
model.pvalues
pd.concat(coef, axis=1)
pd.concat(tvalues, axis=1)
import matplotlib.pyplot as plt

plt.imshow(pd.concat(p, axis=1) > 0.05)
tvalues[0]
r<jupyter_output><empty_output><jupyter_text>Performance & Robustness Check<jupyter_code>Market_cap = get_market_cap_for_period('20101231', '20191231')

equity = get_item_for_period('ifrs_Equity', '20101231', '20191231')

BEME = equity/Market_cap[0]
Market_cap = get_market_cap_for_period('20110630', '20200630')
Market_cap[0]
SMB_mask2 = get_mask(Market_cap[0], Market_cap[1], market_bp ='KOSPI')

SMB2 = get_factor_on_date_by_mask(SMB_mask2, term = 'm', winsorize_limits = 0.01, weight = 'VW')
# SMB
SMB_mask = get_mask(Market_cap[0], Market_cap[1], breakpoint = [(i+1)/10 for i in range(9)])

SMB = get_factor_on_date_by_mask(SMB_mask, term = 'm', winsorize_limits = 0.01, weight = 'VW')
SMB.mean().plot.bar()
import scipy.stats

scipy.stats.ttest_ind(SMB2.iloc[:,0], SMB2.iloc[:,1])
SMB2
t1 = get_market_return_for_period('20010101', '20010104')
t2 = get_market_return_for_period('20050101', '20050104')<jupyter_output><empty_output><jupyter_text>BAB Factor Replication 업데이트 요망  log return overlap할때 가운데 날짜가 들어가는건지? 아니면 앞에 날짜가 들어가는건지? 사실 그리 큰 상관은 없긴 함.  알고리즘 구성  log return 구한 후에, 매 6월마다 종목 선정. 750개 이전이 있으면 체크해서 전부 구함.<jupyter_code>stop = False
flag = find_next_market_date('20000101')
price = []
ind = []
market_ret = []
while not stop:
    end = find_next_market_date(find_next_market_date(find_next_market_date(flag)))
    t = get_market_return_for_period(flag, end)
    price.append(t['등락률'])
    market_ret.append((t['등락률'] * t['시가총액'] / t['시가총액'].sum()).sum())
    flag = find_next_market_date(flag)
    ind.append(flag)
    if flag[6:]=='01':
        print(flag)
    if len(price) == 5338:
        stop = True
with open('price.pkl', 'wb') as f:
    pickle.dump(price, f)
with open('price.pkl', 'rb') as f:
    price = pickle.load(f)
log_return = pd.concat(price, axis=1)
log_return.columns = ind
log_return = log_return.applymap(lambda x: np.log(1+float(x)))
# 기준 날짜 설정 (6월 마지막 거래일)
stand = []
for i in range(log_return.shape[1]-1):
    if log_return.columns[i][4:6]=='06' and log_return.columns[i+1][4:6]=='07':
        stand.append(i)
df = pd.concat([log_return, pd.DataFrame(market_ret, index = log_return.columns, columns = ['market']).T])
corr = []
for i in tqdm.tqdm(range(len(stand)-5)):
    corr.append(df.iloc[:,stand[i]:stand[i+5]].T.corr(min_periods=750).iloc[-1] * (df.iloc[:,stand[i+5]] / df.iloc[:,stand[i+5]]))
    # 뒤의 파트는 해당 시점에 거래되고 있었는지를 체크하기 위한 것.
corr = pd.concat(corr, axis=1)
var = []
bab_ind = []
for i in tqdm.tqdm(range(len(stand)-5)):
    var.append(df.iloc[:,stand[i+4]:stand[i+5]].T.var() * ((df.iloc[:,stand[i+5]]+100) / (df.iloc[:,stand[i+5]]+100)) * ((df.iloc[:,stand[i+5]-120]+100) / (df.iloc[:,stand[i+5]-120]+100)))
    bab_ind.append(df.columns[stand[i+5]])
    # 뒤의 파트는 해당 시점에 거래되고 있었는지를 체크하기 위한 것.
var = pd.concat(var, axis=1)
std = var.applymap(lambda x: np.sqrt(x))
beta = std / std.iloc[-1,:] * corr
# beta 구하기 성공
# beta shrinkage
beta_sh = beta.applymap(lambda x: 0.6*x + 0.4)
beta_sh = beta_sh.iloc[:-1]
beta_sh.columns = bab_ind
k = 2/(beta_sh.rank() - beta_sh.rank().mean()).abs().sum()

w_h = k * ((beta_sh.rank() - beta_sh.rank().mean() > 0) * (beta_sh.rank() - beta_sh.rank().mean()))

w_l = - k * ((beta_sh.rank() - beta_sh.rank().mean() < 0) * (beta_sh.rank() - beta_sh.rank().mean()))

beta_l = (w_l * beta).sum()

beta_h = (w_h * beta).sum()
see = np.array([x-x%3 for x in stand])

r_t = []
for i in range(len(see)-6):
    r_t.append(df.values[:,::3].cumsum(axis=1)[:,int(see[i+6]/3)]- df.values[:,::3].cumsum(axis=1)[:,int(see[i+5]/3)])

r_t = pd.DataFrame(r_t, columns = df.index).T
r_t = r_t.iloc[:-1]

r_l = (r_t * w_l.iloc[:,:-1]).sum()

r_h = (r_t * w_h.iloc[:,:-1]).sum()
bab = (r_l - 0.02) / beta_l - (r_h-0.02)/beta_h
bab
r_l > r_h<jupyter_output><empty_output><jupyter_text>Quality Factor Replication Profitability = gpoa + roe + roa + cfoa + gmar + acc  gpoa = GP / AT  roe = PL / BE  roa = PL / AT  cfoa = FCF / AT = (CFO - PNE_change)/AT = (CFO - CAPEX)/AT (출처: 네이버 금융)  gmar = GP / REVacc = -(WC_change - DP)/AT = -((STR+INV-STP)_change - DP)/ATGrowth = gpoa_change + roe_change + roa_change + cfoa_change + gmar_change  -> 수정주식수로 나누어줘야 하는데, 이걸 어떻게 처리해야 할지 모르겠네...<jupyter_code>GP = get_item_for_period('ifrs_GrossProfit', '20121231', '20201231')
AT = get_item_for_period('ifrs_Assets', '20121231', '20201231')
BE = get_item_for_period('ifrs_Equity', '20121231', '20201231')
DEBT = get_item_for_period('ifrs_Liabilities', '20121231', '20201231')
PL = get_item_for_period('ifrs_ProfitLoss', '20121231', '20201231')
PNE = get_item_for_period('ifrs_PropertyPlantAndEquipment', '20121231', '20201231')
CFO = get_item_for_period('ifrs_CashFlowsFromUsedInOperatingActivities', '20121231', '20201231')
REV = get_item_for_period('ifrs_Revenue', '20121231', '20201231')
DP = get_item_for_period('dart_DepreciationExpense', '20121231', '20201231')
STR = get_item_for_period('dart_ShortTermTradeReceivable', '20121231', '20201231')
STR_2 = get_item_for_period('ifrs_TradeAndOtherCurrentReceivables', '20121231', '20201231')
INV = get_item_for_period('ifrs_Inventories', '20121231', '20201231')
STP = get_item_for_period('dart_ShortTermTradePayables', '20121231', '20201231')
STP_2 = get_item_for_period('ifrs_TradeAndOtherCurrentPayables', '20121231', '20201231')
DP_2 = get_item_for_period('dart_AdjustmentsForDepreciationExpense', '20121231', '20201231')
gpoa = ((GP/AT).rank() - (GP / AT).rank().mean())/(GP/AT).rank().std()
roe = ((PL/BE).rank() - (PL/BE).rank().mean())/(PL/BE).rank().std()
roa = ((PL/AT).rank() - (PL/AT).rank().mean())/(PL/AT).rank().std()
cfoa = (((CFO - (PNE.T - PNE.T.shift(1)).T)/AT).rank() - ((CFO - (PNE.T - PNE.T.shift(1)).T)/AT).rank().mean())/((CFO - (PNE.T - PNE.T.shift(1)).T)/AT).rank().std()
gmar = ((GP/REV).rank() - (GP/REV).rank().mean())/(GP/REV).rank().std()

STR_T = pd.concat([STR, STR_2], axis = 1).groupby(level=0, axis=1).first()
STP_T = pd.concat([STP, STP_2], axis = 1).groupby(level=0, axis=1).first()
WC_change = ((STR_T + INV - STP_T).T - (STR_T + INV - STP_T).T.shift(1)).T
DP_T = t = pd.DataFrame(0, index = WC_change.index, columns = WC_change.columns)
DP_T.loc[list(set(DP_2.index ) & set(WC_change.index)), DP_2.columns] = DP_2.loc[list(set(DP_2.index) & set(WC_change.index)),:].values
acc = ((-(WC_change - DP_T)/AT).rank() - (-(WC_change - DP_T)/AT).rank().mean())/(-(WC_change - DP_T)/AT).rank().std()
P_raw = gpoa + roe + roa + cfoa + gmar + acc
P_score = (P_raw.rank() - P_raw.rank().mean())/P_raw.rank().std()
P_mask = get_mask(P_score, get_market_category(P_score), market_bp = 'KOSPI', breakpoint = [0.3, 0.7])
P_factor = get_factor_on_date_by_mask2(P_mask.iloc[:,1:], term = 'm', winsorize_limits = 0.01, weight = 'VW')
P_factor.mean()<jupyter_output><empty_output><jupyter_text>Growth<jupyter_code>from scipy import stats
def transform_z(data):
    return (data.rank() - data.rank().mean())/data.rank().std()
d_gpoa = transform_z(GP/AT - (GP/AT).T.shift(5).T)
d_roe = transform_z(PL/BE - (PL/BE).T.shift(5).T)
d_roa = transform_z(PL/AT - (PL/AT).T.shift(5).T)
d_cfoa = transform_z(((CFO - (PNE.T - PNE.T.shift(1)).T)/AT) - ((CFO - (PNE.T - PNE.T.shift(1)).T)/AT).T.shift(5).T)
d_gmar = transform_z(GP/REV - (GP/REV).T.shift(5).T)
G_raw = d_gpoa + d_roe + d_roa + d_cfoa + d_gmar
G_score = transform_z(G_raw.iloc[:,-3:])
G_mask = get_mask(G_score, get_market_category(G_score), market_bp = 'KOSPI', breakpoint = [0.3, 0.7])
G_factor = get_factor_on_date_by_mask2(G_mask, term = 'm', winsorize_limits = 0.01, weight = 'VW')
G_factor.mean()
stats.ttest_1samp(G_factor.iloc[:,2], 0)<jupyter_output><empty_output><jupyter_text>Safety<jupyter_code>lev = -transform_z(DEBT/BE)

evol = -transform_z((PL/BE).rolling(5).std().iloc[:,5:])
bab_safety = -transform_z(beta_sh.iloc[:,12:16])

bab_safety.columns = lev.columns[5:]
# altman z-score
WC = STR_T + INV - STP_T
RE = get_item_for_period('ifrs_RetainedEarnings', '20121231', '20201231')
# altman_z = 
RE
S_raw = lev + evol + bab_safety
S_score = transform_z(S_raw)

S_score = S_score.loc[lev.index]
S_mask = get_mask(S_score.iloc[:,5:], get_market_category(S_score.iloc[:,5:]), market_bp = 'KOSPI', breakpoint = [0.3, 0.7])
S_factor = get_factor_on_date_by_mask2(S_mask, term = 'm', winsorize_limits = 0.01, weight = 'VW')
S_factor.mean()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17437456659732223, 'fraction_numerical': 0.048754467381447696, 'mean_word_length': 4.5157399235069136, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '561220', 'license': 'MIT', 'n_tokens_mistral': 8700, 'n_tokens_neox': 8479, 'n_words': 1283, 'path': 'Sirius_API.ipynb', 'repo_name': 'Eternity402/Factor_Investing', 'size': '18747'}