<jupyter_start><jupyter_text>This file just gets the two shards files merges them and saves the basic info to Shards.csv<jupyter_code>from numpy import *
import pandas

# Basic params to set
params = ['vx','vy','vz','sigx','sigy','sigz','population']

# First 30
df1 = pandas.read_csv('../data/GAIA_SDSS_substructures_shards_cands.csv')
names = df1.group_id.unique()

num_cands = size(names)
dat1 = zeros(shape=(num_cands,7))
for i in range(0,num_cands):
    Cand = df1.loc[df1['group_id'] == names[i]]
    U = Cand.GalRVel
    V = Cand.GalTVel
    W = Cand.GalzVel
    dat1[i,0:3] = array([mean(U),mean(V),mean(W)])
    dat1[i,3:6] = array([std(U),std(V),std(W)])
    dat1[i,6] = size(U)
    
df_dat1 = pandas.DataFrame(dat1,columns=params)
df_names = pandas.DataFrame(names,columns=['ID'])
dat1_full = df_names.join(df_dat1)


# Second 30
df2 = pandas.read_csv('../data/GAIA_SDSS_substructures_next_30_cands.csv')
names = df2.group_id.unique()

num_cands = size(names)
dat2 = zeros(shape=(num_cands,7))
for i in range(0,num_cands):
    Cand = df2.loc[df2['group_id'] == names[i]]
    U = Cand.GalRVel
    V = Cand.GalTVel
    W = Cand.GalzVel
    dat2[i,0:3] = array([mean(U),mean(V),mean(W)])
    dat2[i,3:6] = array([std(U),std(V),std(W)])
    dat2[i,6] = size(U)
    
df_dat2 = pandas.DataFrame(dat2,columns=params)
df_names = pandas.DataFrame(names,columns=['ID'])
dat2_full = df_names.join(df_dat2)

# Save basic data
dat_full = dat1_full.append(dat2_full,ignore_index=True)
dat_full.to_csv('../data/Shards.csv',float_format='%.3f',index=False)

# Save everything to one main file
df_full = df1.append(df2,ignore_index=True)[df2.columns.tolist()]
df_full.to_csv('../data/GAIA_SDSS_Shards.csv',float_format='%.8f',index=True)
# Read in basic data to check everything looks right
df = pandas.read_csv('../data/Shards.csv')
names = df.ID
nshards = size(names)
velocities = zeros(shape=(nshards,3))
dispersions = zeros(shape=(nshards,3))
velocities[0:(nshards),0] = df.vx
velocities[0:(nshards),1] = df.vy
velocities[0:(nshards),2] = df.vz
dispersions[0:(nshards),0] = df.sigx
dispersions[0:(nshards),1] = df.sigy
dispersions[0:(nshards),2] = df.sigz
pops = df.population
pops /= sum(pops)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.17776777677767777, 'fraction_numerical': 0.029252925292529253, 'mean_word_length': 5.576923076923077, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '167156', 'license': 'MIT', 'n_tokens_mistral': 968, 'n_tokens_neox': 931, 'n_words': 147, 'path': 'code/SaveShards.ipynb', 'repo_name': 'cajohare/HaloSpin', 'size': '2222'}