<commit_before>import numpy as np
from astLib import astCalc as aca
import h5py as hdf
from sklearn.cross_validation import train_test_split

aca.H0 = 72
aca.OMEGA_M0 = 0.23
aca.OMEGA_L0 = 0.77

def calcMass(vd, A1D = 1082.9, alpha=0.3361):
    avgz = 0.0
    return 1e15/(aca.H0 * aca.Ez(avgz)/100.) * (vd/A1D)**(1/alpha)

f = hdf.File('result_targetedIdeal.hdf5', 'r')
dset  = f[f.keys()[0]]
data = dset.value
mask = (np.log10(data['LOSVD']) > 3.12 ) & (data['M200c'] < 10**14.5)
maskedData = data[~mask]
badData = data[mask]
train, test = train_test_split(maskedData, test_size=0.3)

# make the joint probability
data = np.column_stack((np.log10(train['M200c']), np.log10(train['LOSVD']),
    train['ZSPEC'], np.log10(train['NGAL'])))

Ngrid = 41, 21, 5, 10
grid = [np.linspace(data[:,i].min(), data[:,i].max(), g+1) for i,g in
        enumerate(Ngrid)]

H, (xbins, ybins, zbins, ngbins) = np.histogramdd(data, bins=grid)

# The bin order in the histogram now becomes z, y, x!!!
H = H.T

# Now we try to recover, and predict
expected_mass = np.zeros((test.size,), dtype=[('MASS', '>f4'),<commit_msg>calculated LOSVDs<commit_after>import numpy as np
from astLib import astCalc as aca
import h5py as hdf
from sklearn.cross_validation import train_test_split

aca.H0 = 72
aca.OMEGA_M0 = 0.23
aca.OMEGA_L0 = 0.77

def calcMass(vd, A1D = 1082.9, alpha=0.3361):
    avgz = 0.0
    return 1e15/(aca.H0 * aca.Ez(avgz)/100.) * (vd/A1D)**(1/alpha)

def calcLOSVD(M, z, A1D = 1082.9, alpha=0.3361):
    return A1D * (M * (aca.H0 * aca.Ez(z)/100.)/1e15)**alpha

f = hdf.File('result_targetedIdeal.hdf5', 'r')
dset  = f[f.keys()[0]]
data = dset.value
mask = (np.log10(data['LOSVD']) > 3.12 ) & (data['M200c'] < 10**14.5)
maskedData = data[~mask]
badData = data[mask]
train, test = train_test_split(maskedData, test_size=0.3)

# compute the LOSVD based on the true mass
LOSVD = map(calcLOSVD, train['M200c'], train['ZSPEC'])

LOSVD = [abs(i + i*np.random.normal()*2) for i in LOSVD]


# make the joint probability
data = np.column_stack((np.log10(train['M200c']), np.log10(LOSVD),
    train['ZSPEC'], np.log10(train['NGAL'])))

Ngrid = 41, 21, 5, 10
grid = [np.linspace(data[:,i].min(), data[:,i].max(), g+1) for i,g in
        enumerate(Ngrid)]

H, (xbins, ybins, zbins, ngbins) = np.histogramdd(data, bins=grid)

# The bin order in the histogram now becomes z, y, x!!!
H = H.T

# Now we try to recover, and predict
expected_mass = np.zeros((test.size,), dtype=[('MASS', '>f4'),
{'fraction_non_alphanumeric': 0.18557536466774716, 'fraction_numerical': 0.06523500810372772, 'mean_word_length': 4.450331125827814, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6514564', 'n_tokens_mistral': 1155, 'n_tokens_neox': 1050, 'n_words': 246}