<jupyter_start><jupyter_text>The next part is a little harder. We need to know if there are any sources where the extraction radius of one overlaps with the with extraction radius of another. But we only need to do it in single fields. I think what we should do is load the detection catalogs for each field. Then we check.<jupyter_code>current = ''
for i, row in df_merged.iterrows():
    df_tmp = df_merged.loc[df_merged.Field == row.Field]

    if len(df_tmp) >=2 and row.Field != current:
        current = row.Field
        for x, y in itertools.combinations(df_tmp.index, 2):
            x1 = df_merged.iloc[x].X
            y1 = df_merged.iloc[x].Y
            x2 = df_merged.iloc[y].X
            y2 = df_merged.iloc[y].Y
            r1 = abs(df_merged.iloc[x].EXT_RAD) * 60 / 2.36
            r2 = abs(df_merged.iloc[y].EXT_RAD) * 60 / 2.36
            
            # check to see if one extraction circle wholly inside another
            if (r1 - r2)**2 >= ( (x2-x1)**2 + (y2-y1)**2):
                print(f"{df_merged.iloc[x].Field} {df_merged.iloc[x].INDEX} and {df_merged.iloc[y].INDEX} overlap")
                if not 'B' in df_merged.iloc[x].FLAGS:
                    df_merged.at[x, 'FLAGS'] += 'B'
                if not 'B' in df_merged.iloc[y].FLAGS:
                    df_merged.at[y, 'FLAGS'] += 'B'
            # check if the two extraction circles intersect.
            elif (r1 - r2)**2 <= (x1 - x2)**2 + (y1 - y2)**2 <= (r1 + r2)**2:
                print(f"{df_merged.iloc[x].Field} {df_merged.iloc[x].INDEX} and {df_merged.iloc[y].INDEX} intersect")
                if not 'B' in df_merged.iloc[x].FLAGS:
                    df_merged.at[x, 'FLAGS'] += 'B'
                if not 'B' in df_merged.iloc[y].FLAGS:
                    df_merged.at[y, 'FLAGS'] += 'B'
            else:
                print(f"{df_merged.iloc[x].Field} {df_merged.iloc[x].INDEX} and {df_merged.iloc[y].INDEX} do not intersect")

df_cat = df_cat.merge(df_merged[['Field', 'INDEX', 'FLAGS']],                                                      
               how='left',                                              
               left_on=['NAME', 'INDEX'],                                          
               right_on=['Field', 'INDEX'],                                           
               suffixes=('_cat', '_fit'))
len(df_cat[(df_cat.FLAGS != '') & (~df_cat.DUPLICATE)][['Field', 'REDSHIFT', 'INDEX', 'FLAGS', 'DUPLICATE']])
table = Table.from_pandas(df_cat)
table.filled(-1.0).write('./catalogs/results_table_inspected_flagged_0.5_2.0.fits', overwrite=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1437308868501529, 'fraction_numerical': 0.017201834862385322, 'mean_word_length': 1.8353196099674973, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '215566', 'license': 'MIT', 'n_tokens_mistral': 909, 'n_tokens_neox': 842, 'n_words': 197, 'path': 'paper/05. Flag Known Issues.ipynb', 'repo_name': 'boada/swiftXRT', 'size': '2616'}