<jupyter_start><jupyter_text>Python code (without libraries)  Preparing for Diagnostic Plots<jupyter_code>plt.rc('font', size=14)
plt.rc('figure', titlesize=18)
plt.rc('axes', labelsize=15)
plt.rc('axes', titlesize=18)
def preparing_diag_plots(model_fit):
    # fitted values (need a constant term for intercept)
    model_fitted_y = model_fit.fittedvalues

    # model residuals
    model_residuals = model_fit.resid

    # normalized residuals
    model_norm_residuals = model_fit.get_influence().resid_studentized_internal

    # absolute squared normalized residuals
    model_norm_residuals_abs_sqrt = np.sqrt(np.abs(model_norm_residuals))

    # absolute residuals
    model_abs_resid = np.abs(model_residuals)

    # leverage, from statsmodels internals
    model_leverage = model_fit.get_influence().hat_matrix_diag

    # cook's distance, from statsmodels internals
    model_cooks = model_fit.get_influence().cooks_distance[0]

    return (model_fitted_y, 
        model_residuals, 
        model_norm_residuals,
        model_norm_residuals_abs_sqrt,
        model_abs_resid,
        model_leverage,
        model_cooks)
#     return (model_fitted_y, model_residuals, model_norm_residuals, model_norm_residuals_abs_sqrt, model_abs_resid,model_leverage, model_cooks)
def residuals_vs_fitted(df, model_fitted_y, model_abs_resid, model_residuals):
    plot_lm_1 = plt.figure(1)
    plot_lm_1.set_figheight(8)
    plot_lm_1.set_figwidth(12)

    plot_lm_1.axes[0] = sns.residplot(model_fitted_y, 'y', data=df, 
                              lowess=True, 
                              scatter_kws={'alpha': 0.5}, 
                              line_kws={'color': 'red', 'lw': 1, 'alpha': 0.8})

    plot_lm_1.axes[0].set_title('Residuals vs Fitted')
    plot_lm_1.axes[0].set_xlabel('Fitted values')
    plot_lm_1.axes[0].set_ylabel('Residuals')

    # annotations
    abs_resid = model_abs_resid.sort_values(ascending=False)
    abs_resid_top_3 = abs_resid[:3]

    for i in abs_resid_top_3.index:
        plot_lm_1.axes[0].annotate(i, 
                                   xy=(model_fitted_y[i], 
                                       model_residuals[i]));
def q_q(model_norm_residuals):
    QQ = ProbPlot(model_norm_residuals)
    plot_lm_2 = QQ.qqplot(line='45', alpha=0.5, color='#4C72B0', lw=1)

    plot_lm_2.set_figheight(8)
    plot_lm_2.set_figwidth(12)

    plot_lm_2.axes[0].set_title('Normal Q-Q')
    plot_lm_2.axes[0].set_xlabel('Theoretical Quantiles')
    plot_lm_2.axes[0].set_ylabel('Standardized Residuals');

    # annotations
    abs_norm_resid = np.flip(np.argsort(np.abs(model_norm_residuals)), 0)
    abs_norm_resid_top_3 = abs_norm_resid[:3]

    for r, i in enumerate(abs_norm_resid_top_3):
        plot_lm_2.axes[0].annotate(i, 
                                   xy=(np.flip(QQ.theoretical_quantiles, 0)[r],
                                       model_norm_residuals[i]));
def scale_location(model_fitted_y,model_norm_residuals_abs_sqrt ):
    plot_lm_3 = plt.figure(3)
    plot_lm_3.set_figheight(8)
    plot_lm_3.set_figwidth(12)

    plt.scatter(model_fitted_y, model_norm_residuals_abs_sqrt, alpha=0.5)
    sns.regplot(model_fitted_y, model_norm_residuals_abs_sqrt, 
                scatter=False, 
                ci=False, 
                lowess=True,
                line_kws={'color': 'red', 'lw': 1, 'alpha': 0.8})

    plot_lm_3.axes[0].set_title('Scale-Location')
    plot_lm_3.axes[0].set_xlabel('Fitted values')
    plot_lm_3.axes[0].set_ylabel('$\sqrt{|Standardized Residuals|}$');

    # annotations
    abs_sq_norm_resid = np.flip(np.argsort(model_norm_residuals_abs_sqrt), 0)
    abs_sq_norm_resid_top_3 = abs_sq_norm_resid[:3]

    for i in abs_sq_norm_resid_top_3:
        plot_lm_3.axes[0].annotate(i, 
                                   xy=(model_fitted_y[i], 
                                       model_norm_residuals_abs_sqrt[i]));
def leverage(model_leverage,model_norm_residuals, model_cooks, model_fit ):
    plot_lm_4 = plt.figure(4)
    plot_lm_4.set_figheight(8)
    plot_lm_4.set_figwidth(12)

    plt.scatter(model_leverage, model_norm_residuals, alpha=0.5)
    sns.regplot(model_leverage, model_norm_residuals, 
                scatter=False, 
                ci=False, 
                lowess=True,
                line_kws={'color': 'red', 'lw': 1, 'alpha': 0.8})

    plot_lm_4.axes[0].set_xlim(0, 0.20)
    plot_lm_4.axes[0].set_ylim(-3, 5)
    plot_lm_4.axes[0].set_title('Residuals vs Leverage')
    plot_lm_4.axes[0].set_xlabel('Leverage')
    plot_lm_4.axes[0].set_ylabel('Standardized Residuals')

    # annotations
    leverage_top_3 = np.flip(np.argsort(model_cooks), 0)[:3]

    for i in leverage_top_3:
        plot_lm_4.axes[0].annotate(i, 
                                   xy=(model_leverage[i], 
                                       model_norm_residuals[i]))

    # shenanigans for cook's distance contours
    def graph(formula, x_range, label=None):
        x = x_range
        y = formula(x)
        plt.plot(x, y, label=label, lw=1, ls='--', color='red')

    p = len(model_fit.params) # number of model parameters

    graph(lambda x: np.sqrt((0.5 * p * (1 - x)) / x), 
          np.linspace(0.001, 0.200, 50), 
          'Cook\'s distance') # 0.5 line
    graph(lambda x: np.sqrt((1 * p * (1 - x)) / x), 
          np.linspace(0.001, 0.200, 50)) # 1 line
    plt.legend(loc='upper right');<jupyter_output><empty_output><jupyter_text>Diagnostic plots for firts dataframe<jupyter_code>model = smf.ols(formula='y~x', data=df1)
model_fit = model.fit()
model_fitted_y, model_residuals, model_norm_residuals,model_norm_residuals_abs_sqrt,model_abs_resid,model_leverage,model_cooks = preparing_diag_plots(model_fit)<jupyter_output><empty_output><jupyter_text>1. Residual plot<jupyter_code>residuals_vs_fitted(df=df1, model_abs_resid=model_abs_resid, model_fitted_y=model_fitted_y, model_residuals=model_residuals)<jupyter_output><empty_output><jupyter_text>2. QQ plot<jupyter_code>q_q(model_norm_residuals=model_norm_residuals)<jupyter_output><empty_output><jupyter_text>3. Scale-Location Plot<jupyter_code>scale_location(model_fitted_y=model_fitted_y, model_norm_residuals_abs_sqrt=model_norm_residuals_abs_sqrt)<jupyter_output><empty_output><jupyter_text>4. Leverage plot<jupyter_code>leverage(model_cooks=model_cooks, model_fit=model_fit, model_leverage=model_leverage, model_norm_residuals=model_norm_residuals)<jupyter_output><empty_output><jupyter_text>Diagnostic plots for second dataframe<jupyter_code>model = smf.ols(formula='y~x', data=df4)
model_fit = model.fit()
model_fitted_y, model_residuals, model_norm_residuals,model_norm_residuals_abs_sqrt,model_abs_resid,model_leverage,model_cooks = preparing_diag_plots(model_fit)
residuals_vs_fitted(df=df1, model_abs_resid=model_abs_resid, model_fitted_y=model_fitted_y, model_residuals=model_residuals)
# q_q(model_norm_residuals=model_norm_residuals)
# scale_location(model_fitted_y=model_fitted_y, model_norm_residuals_abs_sqrt=model_norm_residuals_abs_sqrt)
# leverage(model_cooks=model_cooks, model_fit=model_fit, model_leverage=model_leverage, model_norm_residuals=model_norm_residuals)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10991770121355837, 'fraction_numerical': 0.0218998465615846, 'mean_word_length': 3.8445945945945947, 'pattern_counts': {'":': 0, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '631559', 'license': 'MIT', 'n_tokens_mistral': 2857, 'n_tokens_neox': 2668, 'n_words': 329, 'path': 'Homework/Part 1/Homework_2_AnscombeQuartet/AnscombesQuartet.ipynb', 'repo_name': 'lev1khachatryan/ASDS_ML-2', 'size': '7169'}