<jupyter_start><jupyter_text>Taxes<jupyter_code>ie_data = pd.read_csv("../data/ie_data_tax.csv", index_col=False)
ie_data = ie_data[ie_data["Date"]<2011.09]
print(ie_data.columns)
ie_data.head()
preprocess = lambda x: float(x.replace(" ", "").replace(",", "")) if not isinstance(x, float) else x
ie_data["CAPE"] = ie_data["CAPE"].apply(preprocess)
ie_data["Real_Total_Return_Price"] = ie_data["Real_Total_Return_Price"].apply(preprocess)<jupyter_output><empty_output><jupyter_text>Interest Rates<jupyter_code>ie_data.head()
# Calculate 2 and 5 Years Annualized Expected Real Equity Return
n, p = ie_data.shape
return_2 = []
return_5 = []
for i, row in ie_data.iterrows():

    if i+24 < n:
        return_2.append((ie_data.loc[i+24, "Price.1"]/ie_data.loc[i, "Price.1"])**(1/2)-1)
    else:
        return_2.append(np.nan)
    
    if i+60 < n:
        return_5.append((ie_data.loc[i+60, "Price.1"]/ie_data.loc[i, "Price.1"])**(1/5)-1)
    else:
        return_5.append(np.nan)

ie_data["Real_Return_2"] = return_2
ie_data["Real_Return_5"] = return_5

# Calculate 2 and 5 Years Annualized Real Yield of Long-term Bond
yield_2 = []
yield_5 = []
for i, row in ie_data.iterrows():

    if i-24 >= 0:
        yield_2.append(ie_data.loc[i, "Rate GS10"]/100-((ie_data.loc[i, "CPI"]/ie_data.loc[i-24, "CPI"])**(1/2)-1))
    else:
        yield_2.append(np.nan)
    
    if i-60 >= 0:
        yield_5.append(ie_data.loc[i, "Rate GS10"]/100-((ie_data.loc[i, "CPI"]/ie_data.loc[i-60, "CPI"])**(1/5)-1))
    else:
        yield_5.append(np.nan)

ie_data["Real_Yield_2"] = yield_2
ie_data["Real_Yield_5"] = yield_5
# Select varialbles of interest and sample period
df = ie_data[["Date", "Rate GS10", "CAPE", "Yield", "Real Return", "Real Return.1", 
              "Returns.2", "Real_Return_2", "Real_Return_5", "Real_Yield_2", 
              "Real_Yield_5"]].copy()
df.rename(columns={"Rate GS10": "10Y_Nominal_Yield", "Yield": "Excess_CAPE_Yield", "Real Return": "Real_Returns_Equity_10Y", 
                   "Real Return.1": "Real_Returns_Bond_10Y", "Returns.2": "Excess_Real_Returns_Equity_10Y", 
                   "Real_Return_2": "Real_Returns_Equity_2Y", "Real_Return_5": "Real_Returns_Equity_5Y", 
                   "Real_Yield_2": "Real_Yield_2Y", "Real_Yield_5": "Real_Yield_5Y"}, inplace=True)
# Drop NAs, sample period: 1881.01-2011.08
df_clean = df.dropna()
df_clean.head()
df["Log_CAPE"] = np.log(df["CAPE"])
df["CAPE_Inverse"] = 1/df["CAPE"]
df["Real_Yield_10Y"] = df.CAPE_Inverse - df.Excess_CAPE_Yield
# Descriptive statistics
df.describe()<jupyter_output><empty_output><jupyter_text>Regression given by Shiller(1996):$$Real Returns Equity_{t+k}=\alpha+\beta_k \log{CAPE_{t}}+\varepsilon_{t+k,k}$$The intuition behind the regression is that under low-interest rate regimes, CAPE is unusually high. Currently SP500 CAPE sits at 39.6, 2nd highest of all time. A high CAPE ratio suggests that equities are overvalued, resulting in low subsequent returns while a low CAPE is suggestive of higher future returns.<jupyter_code># Regression of different sample period

split = 2007.01

reg_data = df[df.Date < split][["Log_CAPE", "Real_Returns_Equity_10Y"]].dropna()
reg1_before = sm.OLS(reg_data.Real_Returns_Equity_10Y, sm.add_constant(reg_data[["Log_CAPE"]]))
result1_before = reg1_before.fit()

reg_data = df[df.Date >= split][["Log_CAPE", "Real_Returns_Equity_10Y"]].dropna()
reg1_after = sm.OLS(reg_data.Real_Returns_Equity_10Y, sm.add_constant(reg_data[["Log_CAPE"]]))
result1_after = reg1_after.fit()

# Entire period
reg_data = df[["Log_CAPE", "Real_Returns_Equity_10Y"]].dropna()
reg1_all = sm.OLS(reg_data.Real_Returns_Equity_10Y, sm.add_constant(reg_data[["Log_CAPE"]]))
result1_all = reg1_all.fit()

print(result1_after.summary())<jupyter_output>OLS Regression Results                              
===================================================================================
Dep. Variable:     Real_Returns_Equity_10Y   R-squared:                       0.622
Model:                                 OLS   Adj. R-squared:                  0.615
Method:                      Least Squares   F-statistic:                     88.74
Date:                     Wed, 01 Dec 2021   Prob (F-statistic):           5.43e-13
Time:                             22:11:20   Log-Likelihood:                 148.66
No. Observations:                       56   AIC:                            -293.3
Df Residuals:                           54   BIC:                            -289.3
Df Model:                                1                                         
Covariance Type:                 nonrobust                                         
============================================================================[...]
{'fraction_non_alphanumeric': 0.17156137756180012, 'fraction_numerical': 0.04331290935981407, 'mean_word_length': 3.102253032928943, 'pattern_counts': {'":': 9, '<': 16, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '512805', 'license': 'MIT', 'n_tokens_mistral': 1796, 'n_tokens_neox': 1655, 'n_words': 303, 'path': 'notebooks/CAPE Ratio Regression.ipynb', 'repo_name': 'yuchen-robin-cao/Modifying-Shiller-CAPE-Ratio', 'size': '4733'}