<jupyter_start><jupyter_text>Линейная регрессия<jupyter_code>import statsmodels
import scipy as sc
import numpy as np
import pandas as pd
import statsmodels.formula.api as smf
import statsmodels.stats.api as sms
from statsmodels.graphics.regressionplots import plot_leverage_resid2
import matplotlib.pyplot as plt
%pylab inline
def func(x):
    if x['nevermarr'] == 1:
        if pd.notnull(x['heduc']):
            return x
        else:
            x['heduc'] = -1
            return x
    else:
        return x
#foo['z'] = foo.apply(func(row), axis = 1)
#foo<jupyter_output><empty_output><jupyter_text>Посмотрим на распределение целевого признака — уровня заработной платы:<jupyter_code>raw = pd.read_csv('botswana.tsv', sep = '\t', header = 0, parse_dates = [2])
raw.head()
raw['heduc'].isnull().sum()
print raw.shape
data = raw
data['nevermarr'] = data['agefm'].apply(lambda x : 0 if pd.notnull(x) else 1)
data.drop('evermarr', axis=1, inplace=True)
print data['agefm'].isnull().sum()
data['agefm'] = data['agefm'].apply(lambda x : x if pd.notnull(x) else 0)
print data['agefm'].isnull().sum()
#data['heduc'].where(data['nevermarr']  == 1).apply(lambda x : x if pd.notnull(x) else -1 )
data = data.apply(lambda x : func(x), axis = 1)

data['idlnchld_noans'] = data['idlnchld'].apply(lambda x : 0 if pd.notnull(x) else 1)
data['heduc_noans'] = data['heduc'].apply(lambda x : 0 if pd.notnull(x) else 1)
data['usemeth_noans'] = data['usemeth'].apply(lambda x : 0 if pd.notnull(x) else 1)

data['idlnchld'] = data['idlnchld'].apply(lambda x : x if pd.notnull(x) else -1)
data['heduc'] = data['heduc'].apply(lambda x : x if pd.notnull(x) else -2)
data['usemeth'] = data['usemeth'].apply(lambda x : x if pd.notnull(x) else -1)

data = data.dropna(axis=0)
print data.shape[0] * data.shape[1]

string = ""
string = " + ".join(data.columns.values)
print string
print data.head(100)
data['heduc'][:10]
print data['heduc'].isnull().sum()
data.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Int64Index: 4348 entries, 0 to 4360
Data columns (total 18 columns):
ceb               4348 non-null int64
age               4348 non-null int64
educ              4348 non-null object
religion          4348 non-null object
idlnchld          4348 non-null float64
knowmeth          4348 non-null float64
usemeth           4348 non-null float64
agefm             4348 non-null float64
heduc             4348 non-null float64
urban             4348 non-null int64
electric          4348 non-null float64
radio             4348 non-null float64
tv                4348 non-null float64
bicycle           4348 non-null float64
nevermarr         4348 non-null int64
idlnchld_noans    4348 non-null int64
heduc_noans       4348 non-null int64
usemeth_noans     4348 non-null int64
dtypes: float64(9), int64(7), object(2)
memory usage: 611.4+ KB<jupyter_text>Построение модели<jupyter_code>m1 = smf.ols('ceb ~ age + educ + religion + idlnchld + knowmeth + usemeth + agefm + heduc + urban + electric + '\
             'radio + tv + bicycle + nevermarr + idlnchld_noans + heduc_noans + usemeth_noans', 
             data=data)
fitted = m1.fit()
print fitted.summary()
print  sms.het_breushpagan(fitted.resid, fitted.model.exog)
m2 = smf.ols('ceb ~ age + educ + idlnchld + knowmeth + usemeth + agefm + heduc + urban + electric + '\
             'bicycle + nevermarr + idlnchld_noans + heduc_noans + usemeth_noans', 
             data=data)
fitted = m2.fit()
print fitted.summary()
print  sms.het_breushpagan(fitted.resid, fitted.model.exog)
print "F=%f, p=%f, k1=%f" % m1.fit().compare_f_test(m2.fit())
m3 = smf.ols('ceb ~ age + educ + idlnchld + knowmeth + agefm + heduc + urban + electric + '\
             'bicycle + nevermarr + idlnchld_noans + heduc_noans', 
             data=data)
fitted = m3.fit()
print fitted.summary()
print  sms.het_breushpagan(fitted.resid, fitted.model.exog)
m2.fit().compare_f_test(m3.fit())[1]<jupyter_output>OLS Regression Results                            
==============================================================================
Dep. Variable:                    ceb   R-squared:                       0.631
Model:                            OLS   Adj. R-squared:                  0.629
Method:                 Least Squares   F-statistic:                     238.6
Date:                Mon, 27 Jun 2016   Prob (F-statistic):               0.00
Time:                        16:10:07   Log-Likelihood:                -7808.6
No. Observations:                4348   AIC:                         1.568e+04
Df Residuals:                    4316   BIC:                         1.589e+04
Df Model:                          31                                         
Covariance Type:            nonrobust                                         
==================================================================================
                     coef    std err          t [...]
{'fraction_non_alphanumeric': 0.150489396411093, 'fraction_numerical': 0.04567699836867863, 'mean_word_length': 2.456659619450317, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '661793', 'license': 'Apache-2.0', 'n_tokens_mistral': 1783, 'n_tokens_neox': 1642, 'n_words': 397, 'path': 'statistics/Оценка зависимости с помощью регрессии stat.regression.ipynb', 'repo_name': 'Diyago/ML-DL-scripts', 'size': '4904'}