<jupyter_start><jupyter_text>Assembling data for the Hyades<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
import pandas as pd
from astropy.coordinates import SkyCoord
from astropy import units as u
from astropy.io import fits
from tqdm import tqdm

plotpar = {'axes.labelsize': 25,
           'font.size': 25,
           'legend.fontsize': 25,
           'xtick.labelsize': 25,
           'ytick.labelsize': 25,
           'text.usetex': True}
plt.rcParams.update(plotpar)
def convert_hours_to_degrees(df, ra_name, dec_name):
    RA, DE = [], [] #[np.zeros(len(douglas)) for i in range(2)]
    for i in range(len(df)):     
        coords = "{0} {1}".format(df[ra_name].values[i], df[dec_name].values[i])
        c = SkyCoord(coords, unit=(u.hourangle, u.deg))
        RA.append(c.ra.degree)
        DE.append(c.dec.degree)
    df["ra"] = RA
    df["dec"] = DE
    return df<jupyter_output><empty_output><jupyter_text>Douglas rotation periods<jupyter_code>douglas = pd.read_csv("../data/hyades.csv")
m = 0 < douglas.prot.values
plt.plot(douglas.bp[m] - douglas.rp[m], np.log10(douglas.prot[m]), ".")
mass = []
for i, ma in enumerate(douglas.Mass.values):
    try:
        mass.append(float(ma))
    except:
        mass.append(0.)
douglas["mass"] = np.array(mass)
plt.plot(douglas.mass.values[m], np.log10(douglas.prot.values[m]), ".")
plt.gca().invert_xaxis()<jupyter_output><empty_output><jupyter_text>Combine Douglas, Delorme and Radick (still need to add Hartman, Prosser and ASAS).  Delorme<jupyter_code>de = pd.read_csv("../data/hyades_delorme.csv", skiprows=70)
delorme = pd.DataFrame(dict({
                             "prot": de.Per,
                             "J-Ks": de["J-Ks"],
                             "Ks": de.Ksmag,
                             "vmag": de.Vmag,
                             "B-V": de["B-Vl"],
                             "ra": de._RA,
                             "dec": de._DE
                              }))
radick87= pd.read_csv("../data/hyades_radick_1987.csv")
radick95= pd.read_csv("../data/hyades_radick_1995.csv")<jupyter_output><empty_output><jupyter_text>Hartman<jupyter_code>ha = pd.read_csv("../data/hartman.csv", skiprows=76)
ha = convert_hours_to_degrees(ha, "RAJ2000", "DEJ2000")
hartman = pd.DataFrame(dict({"ra": ha.ra, "dec": ha.dec,
                             "jmag": ha.Jmag,
                             "kmag": ha.Kmag,
                             "hmag": ha.Hmag,
                             "vmag": ha.Vmag,
                             "prot": ha.Pe
}))
from astropy.coordinates import FK5

RAJ2000, DEJ2000 = [], []
for i in range(len(radick87)):
    coord = SkyCoord(radick87.RAJ1950.values[i], radick87.decJ1950.values[i], 
                     unit=(u.hourangle, u.deg), equinox="J1950")
    rd = coord.transform_to(FK5(equinox='J2000'))
    RAJ2000.append(rd.ra.degree)
    DEJ2000.append(rd.dec.degree)

radick87["RAJ2000"] = np.array(RAJ2000)
radick87["DEJ2000"] = np.array(DEJ2000)<jupyter_output><empty_output><jupyter_text>Load the gaia crossmatch for the clusters.<jupyter_code>gaia = pd.read_csv("../data/gaia_cluster_crossmatch.csv", skiprows=49)
m = gaia.Cluster.values == "Hyades  "
hg = gaia.iloc[m]
hg.head()
hg["source_id"] = hg.Source<jupyter_output>/Users/ruthangus/anaconda/lib/python3.5/site-packages/ipykernel/__main__.py:5: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy<jupyter_text>And <NAME>'s K2 - Gaia crossmatch.<jupyter_code>bedell = pd.read_csv("K2_c4_gaia.csv")
f = .3
width, height = (max(hg.RA_ICRS) - min(hg.RA_ICRS)), (max(hg.DE_ICRS) - min(hg.DE_ICRS))
plt.figure(figsize=(width*f, height*f))
plt.scatter(hg.RA_ICRS, hg.DE_ICRS, s=30, zorder=0, alpha=.5)
plt.scatter(douglas.ra, douglas.dec, s=25, zorder=1, alpha=.5)
plt.scatter(hartman.ra, hartman.dec, s=25, zorder=2, alpha=.5)
plt.scatter(delorme.ra, delorme.dec, s=25, zorder=3, alpha=.5)
plt.scatter(radick87.RAJ2000, radick87.DEJ2000, zorder=4, alpha=.5)
plt.xlabel("$\mathrm{RA~[Degrees]}$")
plt.ylabel("$\mathrm{Dec~[Degrees]}$")
plt.xlim(48, 87)
plt.ylim(-1, 36)<jupyter_output><empty_output><jupyter_text>Find closest matches between Gaia and other catalogues.<jupyter_code>def find_min_dist(ra1, dec1, ra2, dec2, dist_arcsecs=5.):
    """
    ind1 is a list of indices the same length as ra1 and dec1 that identifies matching stars in ra1 and dec1.
    ind2 is a list of indices the same length as ra2 and dec2 that identifies matching stars in ra2 and dec2.
    """
    closest_ra, closest_dec, distance, ind1, ind2 = [], [], [], [], []
    for i in tqdm(range(len(ra1))):
        e2 = np.sqrt((ra2.values - ra1.values[i])**2 + (dec2.values - dec1.values[i])**2)
        min_distance = min(e2)
        if min_distance < dist_arcsecs/3600.:
            m = e2 == min_distance
            if sum(m) > 1:
                m = np.arange(len(ra2))[m[0]]
            closest_ra.append(ra1.values[i])
            closest_dec.append(dec1.values[i])
            distance.append(min_distance)
            ind1.append(i)
            ind2.append(np.arange(len(ra2))[m])
            
    m1, m2 = np.zeros(len(ra1), dtype="bool"), np.zeros(len(ra2), dtype="bool")
    for i in ind1:
        m1[i] = True
    for i in ind2:
        m2[i] = True
    return m1, m2<jupyter_output><empty_output><jupyter_text>Gaia and Hartman Hartman has J, H and K colors, V-band magnitudes, and the rotation period column is "Pe".<jupyter_code>f = .3
width, height = (max(hg.RA_ICRS) - min(hg.RA_ICRS)), (max(hg.DE_ICRS) - min(hg.DE_ICRS))
plt.figure(figsize=(width*f, height*f))
plt.scatter(hg.RA_ICRS, hg.DE_ICRS, s=30, zorder=0, alpha=.5)
plt.scatter(hartman.ra, hartman.dec, s=25, zorder=1, alpha=.5)
plt.xlabel("$\mathrm{RA~[Degrees]}$")
plt.ylabel("$\mathrm{Dec~[Degrees]}$")
plt.xlim(48, 87)
plt.ylim(-1, 36)<jupyter_output><empty_output><jupyter_text>Find the closest stars<jupyter_code>ind1, ind2 = find_min_dist(hg.RA_ICRS, hg.DE_ICRS, hartman.ra, hartman.dec)
print(sum(ind1), sum(ind2), "matches")
plt.plot(hg.RA_ICRS, hg.DE_ICRS, ".")
plt.plot(hg.RA_ICRS.values[ind1], hg.DE_ICRS.values[ind1], ".", alpha=.5)
plt.xlabel("RA")
plt.ylabel("dec")<jupyter_output><empty_output><jupyter_text>Now add the Hartman catalogue on to the Douglas catalogue.<jupyter_code>hg.iloc[ind1]
hartman.iloc[ind2]
hart_gaia = hartman.iloc[ind2]
hart_gaia["source_id"] = hg.iloc[ind1].source_id.values
df1 = pd.merge(douglas, hart_gaia, how="outer")
df1.prot
print(np.shape(douglas), np.shape(hart_gaia), np.shape(df1))
plt.figure(figsize=(width*f, height*f))
plt.scatter(df1.iloc[:73].ra, df1.iloc[:73].dec, s=30, zorder=0, label="Douglas")
plt.scatter(df1.iloc[73:].ra, df1.iloc[73:].dec, s=30, zorder=1, label="Hartman")
plt.xlabel("$\mathrm{RA~[Degrees]}$")
plt.ylabel("$\mathrm{Dec~[Degrees]}$")
plt.xlim(48, 87)
plt.ylim(-1, 36)
plt.legend()<jupyter_output><empty_output><jupyter_text>Gaia and Delorme Delorme has J-Ks colour, V-band magnitudes and the Period column is called "Per".<jupyter_code>delorme.head()
f = .3
width, height = (max(hg.RA_ICRS) - min(hg.RA_ICRS)), (max(hg.DE_ICRS) - min(hg.DE_ICRS))
plt.figure(figsize=(width*f, height*f))
plt.scatter(hg.RA_ICRS, hg.DE_ICRS, s=30, zorder=0, alpha=.5)
plt.scatter(delorme.ra, delorme.dec, s=25, zorder=1, alpha=.5)
plt.xlabel("$\mathrm{RA~[Degrees]}$")
plt.ylabel("$\mathrm{Dec~[Degrees]}$")
plt.xlim(48, 87)
plt.ylim(-1, 36)

print(len(hg.source_id.values), len(np.unique(hg.source_id.values)))<jupyter_output><empty_output><jupyter_text>Find the closest stars.<jupyter_code>ind1, ind2 = find_min_dist(hg.RA_ICRS, hg.DE_ICRS, delorme.ra, delorme.dec)
print(sum(ind1), sum(ind2), "matches")
plt.plot(hg.RA_ICRS, hg.DE_ICRS, ".")
plt.plot(hg.RA_ICRS.values[ind1], hg.DE_ICRS.values[ind1], ".", alpha=.5)
plt.xlabel("RA")
plt.ylabel("dec")<jupyter_output><empty_output><jupyter_text>Add Delorme to the catalogue.<jupyter_code>de_gaia = delorme.iloc[ind2]
de_gaia["source_id"] = hg.iloc[ind1].source_id.values

df2 = pd.merge((df1, de_gaia), how="outer")
print(np.shape(df1), np.shape(delorme_gaia), np.shape(df2))
plt.figure(figsize=(width*f, height*f))
plt.scatter(df2.iloc[:73].ra, df2.iloc[:73].dec, s=30, zorder=0, alpha=.5, label="Douglas")
plt.scatter(df2.iloc[73:77].ra, df2.iloc[73:77].dec, s=30, zorder=1, alpha=.5, label="Hartman")
plt.scatter(df2.iloc[77:124].ra, df2.iloc[77:124].dec, s=30, zorder=2, alpha=.5, label="Delorme")
plt.xlabel("$\mathrm{RA~[Degrees]}$")
plt.ylabel("$\mathrm{Dec~[Degrees]}$")
plt.xlim(48, 87)
plt.ylim(-1, 36)
plt.legend()
df2.head()
# for i in df2.keys():
#     print(i)
m = df2.prot.values > 0.
plt.plot(df2.bp.values[m] - df2.rp.values[m], df2.prot.values[m], ".")
print(len(df2.prot.values[m]), len(df2))
# print(df2.source_id.values)
source_ids_hyades = pd.DataFrame(dict({"source_id": df2.source_id}))
source_ids_hyades.to_csv("../data/source_ids_hyades.csv")<jupyter_output><empty_output><jupyter_text>After performing the following query with the above data file uploaded:SELECT *FROM user_table_nameJOIN gaiadr2.gaia_source AS gaia USING (source_id)<jupyter_code>gaia_h = pd.read_csv("../data/hyades-result.csv")
gaia_h.head()
df3 = pd.merge(df2, gaia_h, on="source_id")
print(np.shape(df3))
m = 0 < df3.prot.values 
plt.plot(df3.bp[m] - df3.rp[m], df3.prot.values[m], ".")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1758022093634929, 'fraction_numerical': 0.036717517096265126, 'mean_word_length': 3.9769633507853404, 'pattern_counts': {'":': 15, '<': 60, '<?xml version=': 0, '>': 59, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '19460', 'license': 'MIT', 'n_tokens_mistral': 4170, 'n_tokens_neox': 3902, 'n_words': 636, 'path': 'Hyades.ipynb', 'repo_name': 'RuthAngus/extragalactic-exoplanets', 'size': '9511'}