<jupyter_start><jupyter_text>Crossmatching our dippersThis notebook processes the full ZTF dataset to look for dips and creates a new AXS dataset with the results.  Setup<jupyter_code>import getpass
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

import axs
import pyspark.sql.functions as sparkfunc

import dipper
%matplotlib inline
def spark_start(local_dir):
    from pyspark.sql import SparkSession
    
    spark = (
            SparkSession.builder
            .appName("LSD2")
            .config("spark.sql.warehouse.dir", local_dir)
            .config('spark.master', "local[20]")
            .config('spark.driver.memory', '256G') # 128
            .config('spark.local.dir', local_dir)
            .config('spark.memory.offHeap.enabled', 'true')
            .config('spark.memory.offHeap.size', '4G') # 256
            .config("spark.sql.execution.arrow.enabled", "true")
            .config("spark.driver.maxResultSize", "128G")
            .config("spark.driver.extraJavaOptions", f"-Dderby.system.home={local_dir}")
            .enableHiveSupport()
            .getOrCreate()
    )   

    return spark

username = getpass.getuser()
spark_session = spark_start(f"/epyc/users/{username}/spark-tmp/")

catalog = axs.AxsCatalog(spark_session)
spark_session<jupyter_output><empty_output><jupyter_text>https://epyc.astro.washington.edu/jupyter/user/ecbellm/proxy/4049/jobs/<jupyter_code>catalog.list_table_names()<jupyter_output><empty_output><jupyter_text>Match back to class labels from Zooniverse<jupyter_code>dip_cat = catalog.load('wtf_aug2020_dip_candidates')
best_dippers_df = (
    dip_cat
    .exclude_duplicates()
    .sort(sparkfunc.col('dip.asymmetry_significance').desc())
    .toPandas()
)
best_dippers_df
best_dippers_df.columns<jupyter_output><empty_output><jupyter_text>Gaia<jupyter_code>gaia = catalog.load('gaia_source_edr3')
gaia.columns
%%time
dip_x_gaia = dip_cat.crossmatch(gaia, return_min=True).select('ps1_objid','source_id',
'ruwe',  'bp_rp', 'bp_g', 'g_rp',
 'phot_g_mean_flux',
 'phot_g_mean_flux_error',
 'phot_g_mean_mag',
 'phot_bp_mean_flux',
 'phot_bp_mean_flux_error',
 'phot_bp_mean_mag',
 'phot_rp_mean_flux',
 'phot_rp_mean_flux_error',
 'phot_rp_mean_mag',
 'parallax',
 'parallax_error',
 'parallax_over_error',
 'pm',
 'pmra',
 'pmra_error',
 'pmdec',
 'pmdec_error').write.parquet(f'./wtf_aug2020_dip_candidates_x_gaia_source_edr3')
df_dip_x_gaia =  pd.read_parquet('wtf_aug2020_dip_candidates_x_gaia_source_edr3')
df_dip_x_gaia<jupyter_output><empty_output><jupyter_text>Green 19<jupyter_code>g19 = catalog.load("green19_stellar_params")
g19.columns
%%time
dip_x_g19 = dip_cat.crossmatch(g19, return_min=True).select('ps1_objid','dm_16',
        'E_16', 'Mr_16','FeH_16','dm_50','E_50','Mr_50','FeH_50','dm_84', 'E_84',
        'Mr_84', 'FeH_84','chisq','gaia_id').write.parquet(f'./wtf_aug2020_dip_candidates_x_green19_stellar_params')
df_dip_x_g19 =  pd.read_parquet('wtf_aug2020_dip_candidates_x_green19_stellar_params')
df_dip_x_g19<jupyter_output><empty_output><jupyter_text>Bailer-Jones distances<jupyter_code>bjdist = catalog.load('gaia_edr3_distances')
bjdist.columns
%%time
dip_x_bjdist = dip_cat.crossmatch(bjdist, return_min=True).select('ps1_objid','source_id',
'r_med_geo',
 'r_lo_geo',
 'r_hi_geo',
 'r_med_photogeo',
 'r_lo_photogeo',
 'r_hi_photogeo',
 'flag').write.parquet(f'./wtf_aug2020_dip_candidates_x_gaia_edr3_distances')
df_dip_x_bjd =  pd.read_parquet('wtf_aug2020_dip_candidates_x_gaia_edr3_distances')
df_dip_x_bjd<jupyter_output><empty_output><jupyter_text>AllWISE<jupyter_code>wise = catalog.load('allwise_1am_dup')
wise.columns
%%time
dip_x_wise = dip_cat.crossmatch(wise, return_min=True).select('ps1_objid','source_id',
                                                               'w1mpro',
 'w1sigmpro',
 'w1snr',
 'w1rchi2',
 'w2mpro',
 'w2sigmpro',
 'w2snr',
 'w2rchi2',
 'w3mpro',
 'w3sigmpro',
 'w3snr',
 'w3rchi2',
 'w4mpro',
 'w4sigmpro',
 'w4snr',
 'w4rchi2',
 'rchi2',
 'na', 'w1sat',
 'w2sat',
 'w3sat',
 'w4sat',
 'satnum'
).write.parquet(f'./wtf_aug2020_dip_candidates_x_allwise_1am_dup')
df_dip_x_wise =  pd.read_parquet('wtf_aug2020_dip_candidates_x_allwise_1am_dup')
df_dip_x_wise<jupyter_output><empty_output><jupyter_text>Dip labels from Zooniverse Read in the zooniverse labels from zooniverse_analysis.ipynb<jupyter_code>dip_labels = pd.read_csv('ztf_dips_labelled.csv')
dip_labels.head()
dip_labels.dip_label.value_counts()
best_dippers_df = pd.merge(best_dippers_df,dip_labels,left_on='ps1_objid',right_on='ps1_id')
best_dippers_df[['ps1_objid', 'ra', 'dec', 'mean_mag_g', 'mean_mag_r', 'mean_mag_i',
       'ps1_gMeanPSFMag', 'ps1_rMeanPSFMag','ps1_iMeanPSFMag','dip_label']].to_csv('ztf_dips_labelled_allinfo.csv')
best_dippers_df.dip_label.value_counts()
!pwd<jupyter_output>/data/epyc/users/ecbellm/ZTF_Boyajian
{'fraction_non_alphanumeric': 0.1384075907590759, 'fraction_numerical': 0.0297029702970297, 'mean_word_length': 5.848870056497175, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '419765', 'license': 'MIT', 'n_tokens_mistral': 2137, 'n_tokens_neox': 1974, 'n_words': 220, 'path': 'Dippers_crossmatch.ipynb', 'repo_name': 'jradavenport/ZTF_Boyajian', 'size': '4848'}