<jupyter_start><jupyter_text>For empirical part:<jupyter_code>X = pd.read_csv("x_clean.csv", index_col=0)
X.index = range(len(X))
cate = pd.read_csv("CATE average.csv", index_col=0)
cate.index = X.index
progs = ["job search", "vocational", "computer", "language"]
index = pd.MultiIndex.from_product([progs, ["OLS", "perm", "shap"]], names=["program", "metric"])
results = pd.DataFrame(index=index, columns=X.columns)
for prog in progs:
  estimates = cate[prog]
  ols = sm.OLS(estimates, X).fit()
  results.loc[(prog, "OLS")] = ols.params
  boost = xgboost.XGBRegressor(n_estimators=500).fit(X, estimates)
  perm = eli5.sklearn.PermutationImportance(boost, refit=False, random_state=0, n_iter=1)
  perm.fit(X, estimates)
  results.loc[(prog, "perm")] = perm.feature_importances_
  explainer = shap.TreeExplainer(boost)
  shap_values = explainer(X)
  results.loc[(prog, "shap")] = np.abs(shap_values.values).mean(axis=0)
results
results.to_csv("Variable importances.csv")
for prog in progs:
  for metric in ["perm", "shap"]:
    results.loc[(prog, metric)] = results.loc[(prog, metric)] / results.loc[(prog, metric)].max()
table = results[["age", "female", "past_income", "employability", "emp_share_last_2yrs", "foreigner_c", "other_mother_tongue", "unemp_rate", "ue_cw_allocation4"]].round(3).T
table.to_csv("Variable importances formatted.csv")
print(table.round(2).to_latex())
table
import statsmodels.api as sm<jupyter_output><empty_output><jupyter_text>Getting p-values:<jupyter_code>print(sm.OLS(cate["language"], X).fit().summary())<jupyter_output><empty_output><jupyter_text>For DGP 0:<jupyter_code>cate = pd.read_csv("dataset_00.csv", index_col=0)
X = pd.read_csv("X.csv", index_col=0)
estimators = ["R AML", "CF", "BCF", "BART"]
index = pd.MultiIndex.from_product([estimators, ["OLS", "perm", "shap"]], names=["estimator", "metric"])
results = pd.DataFrame(index=index, columns=X.columns)
for estimator in estimators:
  estimates = cate[estimator]
  ols = sm.OLS(estimates, X).fit()
  results.loc[(estimator, "OLS")] = ols.params
  boost = xgboost.XGBRegressor(n_estimators=500).fit(X, estimates)
  perm = eli5.sklearn.PermutationImportance(boost, refit=False, random_state=0, n_iter=1)
  perm.fit(X, estimates)
  results.loc[(estimator, "perm")] = perm.feature_importances_
  explainer = shap.TreeExplainer(boost)
  shap_values = explainer(X)
  results.loc[(estimator, "shap")] = np.abs(shap_values.values).mean(axis=0)
results.to_csv("DGP0 variable importances.csv")
for estimator in estimators:
  for metric in ["perm", "shap"]:
    results.loc[(estimator, metric)] = results.loc[(estimator, metric)] / results.loc[(estimator, metric)].max()
results.T<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17675454883030078, 'fraction_numerical': 0.009654660230226514, 'mean_word_length': 6.970414201183432, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '444401', 'license': 'MIT', 'n_tokens_mistral': 1058, 'n_tokens_neox': 1015, 'n_words': 174, 'path': 'Code/Variable_importances.ipynb', 'repo_name': 'anondissertation/Dissertation', 'size': '2693'}