<jupyter_start><jupyter_text>1. Import<jupyter_code>df = pd.read_stata('Data/USA1950-2016.dta', 
              index_col='Year',)
df.index = pd.to_datetime(df.index, format='%Y')
dfset = df[['realpersonalconsumptionexpenditu','realgdp2009', 'primeratechargedbybanks', 'civilianunemploymentrate']]
dfset['consumption_lag'] = dfset['realpersonalconsumptionexpenditu'].shift(1)
dfset= dfset.reindex(columns= ['realpersonalconsumptionexpenditu', 'consumption_lag','realgdp2009', 'primeratechargedbybanks', 'civilianunemploymentrate'])
dfset.dropna(inplace=True)
dfset.head(2)<jupyter_output>/Users/corybaird/opt/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy<jupyter_text>2. Regression<jupyter_code>x = dfset[['consumption_lag','realgdp2009', 'primeratechargedbybanks', 'civilianunemploymentrate']]
x = np.log(x) #Convert to logs
x = sm.add_constant(x) #Add constant
y = dfset[['realpersonalconsumptionexpenditu']] #Target variable
y = np.log(y) # Convert to log
result = sm.OLS(y, x).fit()
result.summary()<jupyter_output><empty_output><jupyter_text>3.1 Regression 1 lagged: Iterating through variables one by one<jupyter_code>x = dfset[['consumption_lag','realgdp2009', 'primeratechargedbybanks', 'civilianunemploymentrate']]
x = np.log(x) #Convert to logs
x = sm.add_constant(x) #Add constant
x_base = x[['const', 'consumption_lag']]
regressors = ['const']

for i in ['consumption_lag','realgdp2009', 'primeratechargedbybanks', 'civilianunemploymentrate']:
    regressors.append(i)
    result = sm.OLS(y, x[regressors]).fit()
    dft = pd.DataFrame(result.params, columns=['Coeffs'])
    dft['t-test'] = result.tvalues    
    #dft.append(result.rsquared, ignore_index = True) 
    print(dft)
    print('r-squared {:.3f}'.format(result.rsquared))

    print('')<jupyter_output>Coeffs      t-test
const            0.092025    3.355718
consumption_lag  0.992870  303.475144
r-squared 0.999

                   Coeffs     t-test
const           -0.549349  -9.003123
consumption_lag  0.464933   9.596693
realgdp2009      0.571097  10.905882
r-squared 1.000

                           Coeffs     t-test
const                   -0.618605 -10.675737
consumption_lag          0.401253   8.584361
realgdp2009              0.641144  12.662501
primeratechargedbybanks -0.010202  -3.890741
r-squared 1.000

                            Coeffs     t-test
const                    -0.711355 -11.304620
consumption_lag           0.325305   6.384739
realgdp2009               0.721115  13.157427
primeratechargedbybanks  -0.012998  -4.914486
civilianunemploymentrate  0.014376   2.966796
r-squared 1.000<jupyter_text>3.2 Regression 2 lagged: Iterating through variables one by one<jupyter_code>lags = df[['realpersonalconsumptionexpenditu','realgdp2009', 'primeratechargedbybanks', 'civilianunemploymentrate']]
lags['consumption_lag1'] = dfset['realpersonalconsumptionexpenditu'].shift(1)
lags['consumption_lag2'] = dfset['realpersonalconsumptionexpenditu'].shift(2)
lags.dropna(inplace=True)
x = lags[['consumption_lag1','consumption_lag2','realgdp2009', 'primeratechargedbybanks', 'civilianunemploymentrate']]
x = np.log(x) #Convert to logs
x = sm.add_constant(x) #Add constant
y = lags[['realpersonalconsumptionexpenditu']] #Target variable
y = np.log(y) # Convert to log
regressors = ['const','consumption_lag1']

for i in ['consumption_lag2','realgdp2009', 'primeratechargedbybanks', 'civilianunemploymentrate']:
    regressors.append(i)
    result = sm.OLS(y, x[regressors]).fit()
    dft = pd.DataFrame(result.params, columns=['Coeffs'])
    dft['t-test'] = result.tvalues   
    dft['combined']= dft.apply(lambda x:'%s (%s)' % (round(x['Coeffs'],3),round(x['t-test'],3)),axis=1)
    print(dft['combined'])
    #print(dft['combined'].values.reshape(-1,1))
    print('r-squared {:.3f}'.format(result.rsquared))

    print('')<jupyter_output>const                 0.079 (2.619)
consumption_lag1     1.271 (10.379)
consumption_lag2    -0.278 (-2.286)
Name: combined, dtype: object
r-squared 0.999

const               -0.539 (-8.262)
consumption_lag1      0.433 (3.798)
consumption_lag2      0.031 (0.382)
realgdp2009           0.571 (9.896)
Name: combined, dtype: object
r-squared 1.000

const                      -0.605 (-10.329)
consumption_lag1                0.38 (3.81)
consumption_lag2              0.018 (0.253)
realgdp2009                  0.643 (12.218)
primeratechargedbybanks     -0.012 (-4.518)
Name: combined, dtype: object
r-squared 1.000

const                       -0.683 (-10.57)
consumption_lag1              0.419 (4.322)
consumption_lag2            -0.073 (-0.945)
realgdp2009                  0.698 (12.624)
primeratechargedbybanks     -0.014 (-5.267)
civilianunemploymentrate      0.014 (2.451)
Name: combined, dtype: object
r-squared 1.000<jupyter_text>3.2.1 How to save regression into table<jupyter_code>#result.summary()
#with open('summary.csv', 'w') as fh:
    #fh.write(result.summary().as_csv())<jupyter_output><empty_output><jupyter_text>4.1. Serial correlation: Durbin-Watson<jupyter_code>from statsmodels.stats.stattools import durbin_watson

residuals = sm.OLS(y,x).fit().resid
dw_test= durbin_watson(residuals)
if dw_test<1:
    print('Evidence of POSITIVE serial correlation. DW-test stat: {:.3f}'.format(dw_test))
elif dw_test>2:
    print('Evidence of NEGATIVE serial correlation. DW-test stat: {:.3f}'.format(dw_test))
else: 
    print('No evidence of serial correlation. DW-Test stat: {:.3f}'.format(dw_test))<jupyter_output>No evidence of serial correlation. DW-Test stat: 1.044<jupyter_text>4.2. Serial correlation Breusch Godfrey[Explanation via wiki](https://en.wikipedia.org/wiki/Breusch%E2%80%93Godfrey_test)- The null hypothesis is that there is no serial correlation of any order up to p- The test is more general than the Durbin–Watson statistic (or Durbin's h statistic), which is only valid for nonstochastic regressors and for testing the possibility of a first-order autoregressive model (e.g. AR(1)) for the regression errors.- The BG test has none of these restrictions, and is statistically more powerful than Durbin's h statistic.[Statsmodels package](https://www.statsmodels.org/0.8.0/generated/statsmodels.stats.diagnostic.acorr_breusch_godfrey.html)<jupyter_code>from statsmodels.stats.diagnostic import acorr_breusch_godfrey
bg_test = list(acorr_breusch_godfrey(result))
if bg_test[3]<.05:
    print("Reject the null hypothesis that there is no serial correlation. F-stat pvalue of {:.6f}".format(bg_test[3]))<jupyter_output>Reject the null hypothesis that there is no serial correlation. F-stat pvalue of 0.000004
{'fraction_non_alphanumeric': 0.12811180666763722, 'fraction_numerical': 0.07555684961420876, 'mean_word_length': 4.134529147982063, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 25, 'https://': 3, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '416452', 'license': 'CC0-1.0', 'n_tokens_mistral': 2799, 'n_tokens_neox': 2414, 'n_words': 492, 'path': 'Python_basics/Regression_loop.ipynb', 'repo_name': 'corybaird/Econometrics', 'size': '6869'}