<jupyter_start><jupyter_text>Clean DatasetThis notebook generates a table of whether Franzen ATLAS-CDFS objects are extended or compact.<jupyter_code>import astropy.io.ascii as asc, astropy.table, numpy
table = asc.read('one-table-to-rule-them-all.tbl')
sp = table['Component Sp (Franzen)']
s = table['Component S (Franzen)']
franzen_ratios = s / sp
clean = astropy.table.Table(data=[table['Key'], franzen_ratios < 1], names=['Key', 'Clean'], dtype=[int, int])
clean.write('clean-atlas.tbl', format='ascii')
resolved = astropy.table.Table(data=[table['Key'], franzen_ratios > 1], names=['Key', 'Clean'], dtype=[int, int])
resolved.write('resolved-atlas.tbl', format='ascii')
clean
# Analysis: How many of the clean objects have Norris labels?
has_norris_label = [bool(i) and i.startswith('SWIRE') for i in table['Source SWIRE (Norris)']]
print('Compact and Norris-labelled:', numpy.logical_and(franzen_ratios < 1, has_norris_label).sum())
print('Resolved and Norris-labelled:', numpy.logical_and(franzen_ratios > 1, has_norris_label).sum())
# Analysis: How many of the clean objects have RGZ labels?
has_rgz_label = [bool(i) for i in table['Source SWIRE Name (RGZ)']]
print('Compact and RGZ-labelled:', numpy.logical_and(franzen_ratios < 1, has_rgz_label).sum())
print('Resolved and RGZ-labelled:', numpy.logical_and(franzen_ratios > 1, has_rgz_label).sum())
# Analysis: How many of the clean objects have both Norris and RGZ labels?
has_both_label = numpy.logical_and(has_norris_label, has_rgz_label)
print('Compact and labelled:', numpy.logical_and(franzen_ratios < 1, has_both_label).sum())
print('Resolved and labelled:', numpy.logical_and(franzen_ratios > 1, has_both_label).sum())<jupyter_output>Compact and labelled: 332
Resolved and labelled: 188
{'fraction_non_alphanumeric': 0.14367160775370583, 'fraction_numerical': 0.00798175598631699, 'mean_word_length': 7.200934579439252, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '206795', 'license': 'MIT', 'n_tokens_mistral': 644, 'n_tokens_neox': 600, 'n_words': 152, 'path': 'notebooks/103_clean.ipynb', 'repo_name': 'chengsoonong/crowdastro', 'size': '1754'}