<jupyter_start><jupyter_text>1. read the results of u-plot and additional data collection<jupyter_code>#n_train_per_group = 50000

pred_fxn_name = 'rf_classifier'
if pred_fxn_name == 'rf_classifier':
    if results_tag == 'with_demographics':
        max_depth = 16
    else:
        max_depth = 8
    param_dict = {'max_depth': [max_depth], 'n_estimators': [400]}
    num_seeds_eval = 10
    
obj_str = 'ERM'


results_general_path = '../../results/subset_results'
results_descriptor = 'mooc_{0}_{1}_'.format('subsetting', results_tag)
pred_fxn_base_name = 'subset_{0}'.format(group_key, pred_fxn_name)

this_results_path = os.path.join(results_general_path, results_descriptor + obj_str)
results_path_this_pred_fxn = os.path.join(this_results_path,pred_fxn_base_name, pred_fxn_name)
 
these_keys = tuple([x[0] for x in param_dict.values()])
subset_sizes, accs_by_group, accs_total = [], [], []

reload(utils)
# add in all the data sources
subset_types = ['subsetting',
                'additional', 
                'additional_equal_group_sizes',
               ]
num_seeds_by_subset_type = [num_seeds_eval]*3

need_to_tile_data = (len(np.unique(num_seeds_by_subset_type)) == 1)
r = 0
for i,subset_type in enumerate(subset_types):
    results_path_this = results_path_this_pred_fxn.replace('subsetting',subset_type)
    r = utils.read_subset_results_nonimage(results_path_this,
                                           param_dict,
                                           by_seed=True,
                                           seed_start = 0,
                                           num_seeds = num_seeds_by_subset_type[i],
                                           acc_keys = ['acc','auc_roc'])[these_keys] 

    subset_sizes.append(r['subset_sizes'])
    accs_by_group.append(r['accs_by_group'])
    accs_total.append(r['accs_total'])
    
r_both = utils.combine_data_results(subset_sizes,
                                    accs_by_group,
                                    accs_total)
subset_sizes_both, accs_by_group_both, accs_total_both = r_both
# check that the scaling pattern looks right
fig, ax = plt.subplots()
plt.scatter(subset_sizes_both[0], subset_sizes_both[1], 
           c = accs_by_group_both['1 - acc'][0].mean(axis=1))
ax.set_aspect('equal')
ax.set_xlabel(r'\# training samples group A')
ax.set_ylabel(r'\# training samples total')
plt.colorbar(ax=ax)
plt.title('1 - acc')<jupyter_output><empty_output><jupyter_text>2a. fit scaling rules to data<jupyter_code>min_pts_fit = 50

upper_bound_delta = accs_by_group_both[acc_key].min(axis=1).max()
popts, pcovs = fit_scaling_law.get_group_fits(group_pair = group_pair,
                                              accs_by_group = accs_by_group_both,
                                              subset_sizes = subset_sizes_both,
                                              acc_key = acc_key,
                                              delta_bounds = [0,upper_bound_delta],
                                              min_pts = min_pts_fit,
                                              # already tiled
                                              need_to_tile_data=need_to_tile_data)

# print line for putting in the overleaf
group_pair_print = [r'edu $\leq 2^{\circ}$',r'edu $> 2^{\circ}$']

p.print_table_rows('Mooc '+ results_tag,group_pair_print, popts, pcovs, min_pts_fit)<jupyter_output><empty_output><jupyter_text>2b. plot the scaling law fits<jupyter_code>num_seeds_eval = 10
for i,subset_type in enumerate(['additional_equal_group_sizes']):
    
    results_path_this = results_path_this_pred_fxn.replace('subsetting',subset_type)
    r = utils.read_subset_results_nonimage(results_path_this,
                                           param_dict,
                                           by_seed=True,
                                           seed_start = 0,
                                           num_seeds = num_seeds_eval,
                                           acc_keys = ['acc','auc_roc'])[these_keys] 

    subset_sizes_plot = r['subset_sizes']
    accs_by_group_plot = r['accs_by_group']
    accs_total_plot = r['accs_total']
        
fig, ax = p.setup_scaling_plot_ax()
reload(p)
p.plot_scaling_fits(subset_sizes_plot,
                                accs_by_group_plot, 
                                group_names = group_pair,
                                n_thresh_for_scaling = min_pts_fit,
                                n_thresh_for_plotting = 0,
                                acc_key = acc_key, 
                                popts=popts,
                                loglog=True, 
                                show_data_not_fitted = True,
                                show_fitted_line = True,
                                max_one_group=False,
                                dot_legend = True,
                                full_legend = True,
                                ax=ax)
ax.set_ylim(.015)
ax.set_xlim(25)
ax.set_xlabel(r'\# training points from each group ($n_A = n_B$)');
ax.set_title('Mooc')

plt.savefig('../../figures/scaling_mooc_{0}_big_legend.pdf'.format(results_tag), bbox_inches='tight')
fig, ax = p.setup_scaling_plot_ax()
reload(p)
p.plot_scaling_fits(subset_sizes_plot,
                                accs_by_group_plot, 
                                group_names = group_pair,
                                n_thresh_for_scaling = min_pts_fit,
                                n_thresh_for_plotting = 0,
                                acc_key = acc_key, 
                                popts=popts,
                                loglog=True, 
                                show_data_not_fitted = True,
                                show_fitted_line = True,
                                max_one_group=False,
                                dot_legend = True,
                                full_legend = False,
                                ax=ax)
ax.set_ylim(.015)
ax.set_xlim(25)
ax.set_xlabel(r'\# training points from each group ($n_A = n_B$)');
ax.set_title('Mooc')

plt.savefig('../../figures/scaling_mooc_{0}_small_legend.pdf'.format(results_tag), bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>extra: plot 3d<jupyter_code>%pylab widget


plt.rcParams.update({"text.usetex": False})


show_fitted_line = True

for g in [0,1]:
    plt.figure(figsize=(10,8))
    ax = plt.axes(projection='3d')
    
    if need_to_tile_data:
        ns, y = fit_scaling_law.tile_data(subset_sizes_both.sum(axis=0), 
                          accs_by_group_both[acc_key][g])

        njs, _ = fit_scaling_law.tile_data(subset_sizes_both[g], 
                          accs_by_group_both[acc_key][g])
        
        y_means = accs_by_group_both[acc_key][g].mean(axis=1)
        ax.scatter3D(subset_sizes_both.sum(axis=0), subset_sizes_both[g], 
                y_means, c=y_means, s=100,cmap='inferno')
        

    else:
        ns = subset_sizes_both.sum(axis=0)
        njs = subset_sizes_both[g]
        y = accs_by_group_both[acc_key][g]
        
        ax.scatter3D(subset_sizes_both.sum(axis=0), subset_sizes_both[g], 
                y, c=y, s=100,cmap='inferno')
            
    
    ax.set_xlabel('n')
    ax.set_ylabel('n_j')
    ax.set_zlabel('err')

    if show_fitted_line:
        x_fit, y_fit = np.meshgrid(np.linspace(ns.min(), ns.max(),100),
                                   np.linspace(njs.min(), njs.max(),50))
        
        def f_fit(x_0,y_0):
            return fit_scaling_law.modified_ipl((x_0, y_0), *popts[g])
        
        z_fit = f_fit(y_fit, x_fit)
        
        ax.plot_surface(x_fit, y_fit, z_fit, 
                        rstride=1, cstride=1,cmap='inferno', 
                        edgecolor='none',
                        alpha=0.5,
                        zorder=3)
        
        ax.set_title(group_id_dict[g])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09920182440136831, 'fraction_numerical': 0.011022424933485367, 'mean_word_length': 1.7698245614035089, 'pattern_counts': {'":': 1, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 37, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '71904', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2623, 'n_tokens_neox': 2481, 'n_words': 349, 'path': 'mooc/notebooks/fit_scaling_law_mooc.ipynb', 'repo_name': 'estherrolf/representation-matters', 'size': '7893'}