<jupyter_start><jupyter_text>Regressions for Quality Filter Analysis<jupyter_code>%load_ext autoreload
%autoreload 2
%matplotlib inline
sns.set(context='paper', style='white', font_scale=1.3)
import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)
pd.options.mode.chained_assignment = None  # default='warn'<jupyter_output>The autoreload extension is already loaded. To reload it, use:
  %reload_ext autoreload<jupyter_text>Load Model<jupyter_code>from lr.eval import load_model
clf, clf_vectorizer = load_model("../logistic_regression/new_model/")<jupyter_output><empty_output><jupyter_text>Load and Featurize Data<jupyter_code>from data.loader import load_school_news_data, get_high_schools, compute_stats
from data.featurize import demographic_featurize, preprocess
original_df = pd.read_json("../logistic_regression/school_newspapers/articles_with_scores.jsonl", lines=True)
df = load_school_news_data()
high_schools = get_high_schools(df)
stats = compute_stats(high_schools)
stats
features = demographic_featurize(high_schools)
features.shape<jupyter_output><empty_output><jupyter_text>Document Feature Exploration<jupyter_code>from data.loader import load_and_score
from data.constants import DATA_DIR
from data.cluster import cluster_text
from data.viz import build_school_category_plot
high_news = load_and_score(path=DATA_DIR / "articles-high-reliability-clean.jsonl")
build_school_category_plot(high_schools, high_news, save=True, save_path='highschool_boxplot.pdf')
# sample 10K opinion articles
opeds = high_schools.loc[high_schools.category.apply(lambda x: "opinion" in x)].sample(10000).copy().reset_index()

# cluster text via prob high quality
opeds = cluster_text(opeds,
                     num_clusters=10,
                     num_words=10,
                     plot_boxplot=True,
                     save=True,
                     save_path="topic_boxplot.pdf")<jupyter_output><empty_output><jupyter_text>Document Regression<jupyter_code>from data.featurize import text_featurize
text_features = text_featurize(opeds)
formula = "prob_high_quality~ 1 + num_tokens + first_person + third_person + topic" 
doc_model = smf.ols(formula=formula, data=text_features).fit()
print(doc_model.summary())<jupyter_output>OLS Regression Results                            
==============================================================================
Dep. Variable:      prob_high_quality   R-squared:                       0.332
Model:                            OLS   Adj. R-squared:                  0.331
Method:                 Least Squares   F-statistic:                     414.2
Date:                Sat, 22 Jan 2022   Prob (F-statistic):               0.00
Time:                        19:26:15   Log-Likelihood:                 1749.4
No. Observations:               10000   AIC:                            -3473.
Df Residuals:                    9987   BIC:                            -3379.
Df Model:                          12                                         
Covariance Type:            nonrobust                                         
========================================================================================
                           coef    std err[...]<jupyter_text>Demographic Feature Exploration<jupyter_code>from data.viz import build_correlation_plots
features_ = preprocess(features, REGRESSION_FEATURES, impute_only=True)
build_correlation_plots(features_, save=True, save_path='scatters.pdf')<jupyter_output><empty_output><jupyter_text>Demographic Regressions<jupyter_code>import statsmodels.formula.api as smf
from data.constants import REGRESSION_FEATURES

features_preprocessed = preprocess(features, REGRESSION_FEATURES)
features_preprocessed.shape
features_preprocessed.base_county.value_counts()
formula = """
        prob_high_quality ~ 
        1 +
        pupil_ratio +
        school_size + 
        pct_rural +
        is_charter + 
        is_magnet  + 
        is_public + 
        num_degree_holders + 
        median_home_value 
        """

model = smf.ols(formula=formula, data=features_preprocessed).fit()
print(model.summary())
formula = """
        prob_high_quality ~ 
        1 +
        pupil_ratio +
        school_size + 
        pct_rural +
        is_charter + 
        is_magnet  + 
        is_public + 
        asian_ratio + 
        mixed_ratio + 
        black_ratio + 
        hispanic_ratio +
        num_degree_holders + 
        median_home_value 
        """

model = smf.ols(formula=formula, data=features_preprocessed).fit()
print(model.summary())
formula = """
        prob_high_quality ~ 
        1 +
        pupil_ratio +
        school_size + 
        pct_rural +
        is_charter + 
        is_magnet  + 
        is_public + 
        num_degree_holders + 
        median_home_value  + 
        rep_share_2016
        """

model = smf.ols(formula=formula, data=features_preprocessed).fit()
print(model.summary())
formula = """
        prob_high_quality ~ 
        1 +
        mixed_ratio +
        hispanic_ratio +
        asian_ratio +
        black_ratio +
        native_ratio +
        pupil_ratio +
        pacific_islander_ratio +
        school_size + 
        urban_top_30  +
        rural_top_30 +
        is_charter + 
        is_magnet  + 
        is_private + 
        num_degree_holders + 
        median_home_value +
        rep_share_2016
        """

model = smf.ols(formula=formula, data=features_).fit()
print(model.summary())
from scipy import stats
sns.regplot(features_['fte_teachers'], features_['school_size'])<jupyter_output><empty_output><jupyter_text>TOEFL Exam Regression<jupyter_code>toefl_exams = load_and_score(DATA_DIR / 'toefl.jsonl')
model = "C(score_level, Treatment(reference='medium')) + prompt"
formula = "prob_high_quality~ 1 + " + model
#formula = "log_prob_high_quality ~ 1  +  pop_urban + fte_teacher + dem_share  + proportion_white + pupil_ratio + income + asian  + mixed   + black"
model = smf.ols(formula=formula, data=toefl_exams).fit()
print(model.summary())
toefl_exams.groupby(['prompt', 'prompt_text']).prob_high_quality.mean().sort_values()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11203252032520325, 'fraction_numerical': 0.013658536585365854, 'mean_word_length': 2.895503483217226, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '390408', 'license': 'Apache-2.0', 'n_tokens_mistral': 1999, 'n_tokens_neox': 1849, 'n_words': 310, 'path': 'Regressions.ipynb', 'repo_name': 'kernelmachine/quality-filter', 'size': '6150'}