<jupyter_start><jupyter_text>Examining Volunteer internal consistency<jupyter_code>%load_ext autoreload
%autoreload 2
import json
import os
import re
import numpy as np
import scipy.stats as st
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import lib.galaxy_utilities as gu
from tqdm import tqdm
from functools import partial
from gzbuilder_analysis.aggregation.spirals.oo import Arm
import gzbuilder_analysis.parsing as pg
# %run get_duplicate_galaxies.py
duplicates = pd.read_csv('lib/duplicate_galaxies.csv', index_col=0)
ss_ids, validation_ids = duplicates.values.T.astype(int)
dr8ids = duplicates.index.values.astype(int)
def get_models(sid):
    return gu.classifications.query(
        'subject_ids == {}'.format(sid)
    ).apply(pg.parse_classification, image_size=(512, 512), axis=1)<jupyter_output><empty_output><jupyter_text>How in-agreement were our volunteers? These plots show the variance in the percentage of volunteers drawing a component for galaxies in our original and validation subsets, including binomial error estimates.The spiral arm plot shows the mean number of spiral arms for each galaxy, for which it is difficult to determine an error.<jupyter_code>df
df = {}
with tqdm(range(len(dr8ids))) as bar:
    for i in bar:
        id_details = {
            'original_id': ss_ids[i],
            'validation_id': validation_ids[i],
            'dr8id': dr8ids[i],
        }
        ss_models = get_models(ss_ids[i]).apply(pd.Series)
        val_models = get_models(validation_ids[i]).apply(pd.Series)

        ss_counts = ss_models.notna().sum(axis=0)
        val_counts = val_models.notna().sum(axis=0)
        # assume binomial error
        n_ss_model = len(ss_models)
        n_val_model = len(val_models)

        ss_err = np.sqrt(ss_counts * (1 - ss_counts / n_ss_model)) / n_ss_model
        val_err = np.sqrt(val_counts * (1 - val_counts / n_val_model)) / n_val_model

        ss_spiral = ss_models.spiral.apply(lambda a: a.get('n_arms', 0))
        val_spiral = val_models.spiral.apply(lambda a: a.get('n_arms', 0))
        
        df[i] = pd.Series({
            **id_details,
            **ss_counts.add_prefix('ss_').add_suffix('_frac') / len(ss_models),
            **val_counts.add_prefix('val_').add_suffix('_frac') / len(val_models),
            **ss_err.add_prefix('ss_').add_suffix('_err') ,
            **val_err.add_prefix('val_').add_suffix('_err'), 
            'ss_spiral_frac': ss_spiral.mean(),
            'val_spiral_frac': val_spiral.mean(),
            # this does not provide a good measure of error, as our numbers are not Normally distributed
#             'ss_spiral_err': ss_spiral.std(),
#             'val_spiral_err': val_spiral.std(),
        })
    df = pd.DataFrame(df).T
fig, axs = plt.subplots(ncols=4, figsize=(16, 5), dpi=100)
for i, (k, ax) in enumerate(zip(('disk', 'bulge', 'bar', 'spiral'), axs)):
    ax.errorbar(
        df[f'ss_{k}_frac'], df[f'val_{k}_frac'],
        xerr=df.get(f'ss_{k}_err', 0), yerr=df.get(f'val_{k}_err', 0),
        fmt='.', c=f'C{i}', elinewidth=0.5, capsize=1
    )
(ax_disk, ax_bulge, ax_bar, ax_spiral) = axs
for ax in (ax_disk, ax_bulge, ax_bar):
    ax.set_xlabel('Original set')
    ax.set_xlim(-0.1, 1.1)
    ax.set_ylim(-0.1, 1.1)
    ax.set_aspect('equal')
ax_disk.set_ylabel('Validation set')
ax_spiral.set_xlabel('Original set')
ax_bulge.set_yticks([])
ax_bar.set_yticks([])
ax_spiral.set_xlim(-0.1, 3)
ax_spiral.set_ylim(-0.1, 3)
ax_spiral.set_aspect('equal')

ax_disk.set_title('Fraction of classifications with Disc')
ax_bulge.set_title('Fraction of classifications with Bulge')
ax_bar.set_title('Fraction of classifications with Bar')
ax_spiral.set_title('Mean number of spiral arms drawn')


for a in (ax_disk, ax_bulge, ax_bar, ax_spiral):
    a.add_line(plt.Line2D((-1E2, 1E2), (-1E2, 1E2), c='k', alpha=0.2))
plt.savefig('method-paper-plots/component_frequency.pdf', bbox_inches='tight')<jupyter_output><empty_output><jupyter_text>The aggregate modelHow consistent is our aggregated model? We explore the consistency with which a component appears in our aggregated model, and how frequently we obtain a consistent number of spiral arms.<jupyter_code>from gzbuilder_analysis import load_aggregation_results
agg_results = load_aggregation_results('output_files/aggregation_results')
disks = agg_results.apply(lambda a: a.model.get('disk', pd.Series([], dtype=float))).apply(pd.Series)
bulges = agg_results.apply(lambda a: a.model.get('bulge', pd.Series([], dtype=float))).apply(pd.Series)
bars = agg_results.apply(lambda a: a.model.get('bar', pd.Series([], dtype=float))).apply(pd.Series)
spirals = agg_results.apply(lambda a: a.spiral_arms)

disks_e = agg_results.apply(lambda a: a.errors.get('disk', pd.Series([], dtype=float))).apply(pd.Series)
bulges_e = agg_results.apply(lambda a: a.errors.get('bulge', pd.Series([], dtype=float))).apply(pd.Series)
bars_e = agg_results.apply(lambda a: a.errors.get('bar', pd.Series([], dtype=float))).apply(pd.Series)
disks_agree = ~np.logical_xor(
    disks['Re'].loc[ss_ids].reset_index(drop=True).isna(),
    disks['Re'].loc[validation_ids].reset_index(drop=True).isna()
)
bulges_agree = ~np.logical_xor(
    bulges['Re'].loc[ss_ids].reset_index(drop=True).isna(),
    bulges['Re'].loc[validation_ids].reset_index(drop=True).isna()
)
bars_agree = ~np.logical_xor(
    bars['Re'].loc[ss_ids].reset_index(drop=True).isna(),
    bars['Re'].loc[validation_ids].reset_index(drop=True).isna()
)

n_spirals_original = spirals.apply(len).loc[ss_ids].reset_index(drop=True)
n_spirals_validation = spirals.apply(len).loc[validation_ids].reset_index(drop=True)

n_spirals_agree = n_spirals_original == n_spirals_validation
n_spirals_within_1 = (n_spirals_original - n_spirals_validation).abs() < 2

print('Disk agrees {:.3%} of the time'.format(disks_agree.sum() / len(disks_agree)))
print('Bulge agrees {:.3%} of the time'.format(bulges_agree.sum() / len(disks_agree)))
print('Bar agrees {:.3%} of the time'.format(bars_agree.sum() / len(disks_agree)))
print('N_spirals agree {:03.2%} of the time'.format(
    n_spirals_agree.sum() / len(n_spirals_agree)
))
print('N_spirals within 1 {:03.2%} of the time'.format(
    n_spirals_within_1.sum() / len(n_spirals_within_1)
))
print('Total model agrees {:.3%} of the time'.format(
    (disks_agree & bulges_agree & bars_agree & n_spirals_agree).sum() / len(disks_agree)
))
n_spirals_original = spirals.apply(len).loc[ss_ids]
n_spirals_validation = spirals.apply(len).loc[validation_ids]

n_spirals_agree = n_spirals_original == n_spirals_validation.values
n_spirals_within_1 = np.abs(n_spirals_original - n_spirals_validation.values) < 2
print('N_spirals agree {:03.2%} of the time'.format(
    n_spirals_agree.sum() / len(n_spirals_agree)
))
print('N_spirals within 1 {:03.2%} of the time'.format(
    n_spirals_within_1.sum() / len(n_spirals_within_1)
))<jupyter_output>N_spirals agree 53.06% of the time
N_spirals within 1 92.86% of the time<jupyter_text>How consistent are the isophotes for our aggregated shapes?<jupyter_code>fig, axes = plt.subplots(nrows=2, ncols=3, figsize=(15, 10))
ax_disk, ax_bulge, ax_bar = np.array(axes).T
# Disk
ax = ax_disk
ax[0].errorbar(
    disks['q'].loc[ss_ids],
    disks['q'].loc[validation_ids],
    xerr=disks_e['q'].loc[ss_ids],
    yerr=disks_e['q'].loc[validation_ids],
    fmt='.', c='C0', elinewidth=0.5, capsize=1
)
ax[1].errorbar(
    disks['Re'].loc[ss_ids] * 0.396,
    disks['Re'].loc[validation_ids] * 0.396,
    xerr=disks_e['Re'].loc[ss_ids] * 0.396,
    yerr=disks_e['Re'].loc[validation_ids] * 0.396,
    fmt='.', c='C0', elinewidth=0.5, capsize=1
)
ax[0].set_title('Disc')

ax[0].set_xlabel('Axis Ratio (Original subset)')
ax[1].set_xlabel('Size (Original subset), arcseconds')
ax[0].set_ylabel('Axis Ratio (Validation subset)')
ax[1].set_ylabel('Size (Validation subset), arcseconds')

# Bulge
ax = ax_bulge
ax[0].errorbar(
    bulges['q'].loc[ss_ids],
    bulges['q'].loc[validation_ids],
    xerr=bulges_e['q'].loc[ss_ids],
    yerr=bulges_e['q'].loc[validation_ids],
    fmt='.', c='C1', elinewidth=0.5, capsize=1
)
ax[1].errorbar(
    bulges['Re'].loc[ss_ids] * 0.396,
    bulges['Re'].loc[validation_ids] * 0.396,
    xerr=bulges_e['Re'].loc[ss_ids] * 0.396,
    yerr=bulges_e['Re'].loc[validation_ids] * 0.396,
    fmt='.', c='C1', elinewidth=0.5, capsize=1
)
ax[0].set_title('Bulge')
ax[0].set_xlabel('Axis Ratio (Original subset)')
ax[1].set_xlabel('Size (Original subset), arcseconds')


# Bar
ax = ax_bar
ax[0].errorbar(
    bars['q'].loc[ss_ids],
    bars['q'].loc[validation_ids],
    xerr=bars_e['q'].loc[ss_ids],
    yerr=bars_e['q'].loc[validation_ids],
    fmt='.', c='C2', elinewidth=0.5, capsize=1
)
ax[1].errorbar(
    bars['Re'].loc[ss_ids] * 0.396,
    bars['Re'].loc[validation_ids] * 0.396,
    xerr=bars_e['Re'].loc[ss_ids] * 0.396,
    yerr=bars_e['Re'].loc[validation_ids] * 0.396,
    fmt='.', c='C2', elinewidth=0.5, capsize=1
)
ax[0].set_title('Bar')

ax[0].set_xlabel('Axis Ratio (Original subset)')
ax[1].set_xlabel('Size (Original subset), arcseconds')
import matplotlib.ticker as mticker
for ax in (ax_disk, ax_bulge, ax_bar):
    ax[1].set_xscale('log')
    ax[1].set_yscale('log')
    for a in ax:
        l = a.get_xlim() + a.get_ylim()
        lims = min(l), max(l)
        a.add_line(plt.Line2D((-1e3, 1e3), (-1e3, 1e3), c='k', alpha=0.2))
        a.set_xlim(lims); a.set_ylim(lims)
    ax[0].set_xlim(0, 1.1)
    ax[0].set_ylim(0, 1.1)
plt.savefig('method-paper-plots/component_sizing.pdf', bbox_inches='tight')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16321622751986617, 'fraction_numerical': 0.02164366373902133, 'mean_word_length': 4.7309766327142, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '597232', 'license': 'MIT', 'n_tokens_mistral': 3831, 'n_tokens_neox': 3651, 'n_words': 657, 'path': 'comparison__validation_subset.ipynb', 'repo_name': 'tingard/gzbuilder_results', 'size': '9564'}