<jupyter_start><jupyter_text>dataset:**Lens**```/data/inspur_disk03/userdir/wangcx/lens_cxchange to /data/inspur_disk03/userdir/wangcx/lens_cx_add_bass @0822```**non-lens**```/data/inspur_disk03/userdir/wangcx/decals_non_lens_BASS_resulation/PSF/101t101_good/data/inspur_disk03/userdir/wangcx/decals_non_lens_BASS_resulation/COMP/101t101_good/data/inspur_disk03/userdir/wangcx/decals_non_lens_BASS_resulation/DEV/101t101_good/data/inspur_disk03/userdir/wangcx/decals_non_lens_BASS_resulation/REX/101t101_good```<jupyter_code>import astropy.io.fits as fits
import numpy as np
import os
import glob
import tqdm
import matplotlib.pylab as plt
def check_dir(path):
    if not os.path.isdir(path):
        print('mkdir: ', path)
        os.makedirs(path)
DirBase = "/data/inspur_disk03/userdir/wangcx"
OutBase = "/data/dell5/userdir/maotx/Lens/0822"
check_dir(OutBase)
data_shape = [101,101,3]
fp_lens = glob.glob(os.path.join(DirBase,'lens_cx_add_bass/cutout*.fits'))
fp_nlens = glob.glob(os.path.join(DirBase,'decals_non_lens_BASS_resulation/*/101t101_good/cutout*.fits'))
fp_lens=[i.replace(DirBase+'/','') for i in fp_lens]
fp_nlens=[i.replace(DirBase+'/','') for i in fp_nlens]
def show(data):
    fig, axes = plt.subplots(1,4,figsize=(4.5*4,4.5))
    ax = axes[0]
    print(data.shape)
    ax.imshow(data[:,:,0])
    ax.invert_yaxis()
    ax = axes[1]
    ax.imshow(data[:,:,1])
    ax.invert_yaxis()
    ax = axes[2]
    ax.imshow(data[:,:,2])
    ax.invert_yaxis()
    ax = axes[3]
    ax.imshow(data)
    ax.invert_yaxis()
def preprocess(func):
    def wrapper(*args, **kwargs):
        fp = args[0]
        data = func(fp).astype(np.float32)
        cut = 100
        #---------------------------
        m = data.mean(axis=(1,2))[:,None,None]
        s = data.std(axis=(1,2))[:,None,None]
        data -= m
        data /= s
        data = data.transpose(1,2,0)
        #data = np.clip(data,-cut,cut)
        #---------------------------
        return data
    return wrapper
@preprocess
def readdata(fp):
    with fits.open(fp) as hdu:
        data = hdu[0].data
    return data
data = readdata(os.path.join(DirBase,fp_lens[0]))
show(data)
data = readdata(os.path.join(DirBase,fp_lens[2]))
show(data)
data = readdata(os.path.join(DirBase,fp_lens[4]))
show(data)
data = readdata(os.path.join(DirBase,fp_nlens[0]))
show(data)<jupyter_output>Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).
Clipping input data to the valid range for imshow with RGB data ([0..1] for floats or [0..255] for integers).<jupyter_text>select training sample<jupyter_code>frac = np.array([0.8,0.9,1.0])
frac = np.array([0.9,1.0,1.0])
dt = np.dtype([('label', np.int32, 1), ('image', np.float32, data_shape)])
def sel(size, frac):
    random_ind = np.random.permutation(np.arange(size))
    ind_dataset = np.ceil(frac*size).astype(np.int32)
    ind_train = random_ind[:ind_dataset[0]]
    ind_valid = random_ind[ind_dataset[0]:ind_dataset[1]]
    ind_test = random_ind[ind_dataset[1]:ind_dataset[2]]
    return ind_train, ind_valid, ind_test
def get_filename(fps, frac):
    index = sel(len(fps), frac)
    fp_tr = [fps[i] for i in index[0]]
    fp_va = [fps[i] for i in index[1]]
    fp_te = [fps[i] for i in index[2]]
    return fp_tr, fp_va, fp_te
path = os.path.join(OutBase, 'data')
check_dir(path)
FNs = {}
for fps, ns in zip([fp_lens, fp_nlens], ['lens', 'nlens']):
    fp_tr, fp_va, fp_te = get_filename(fps, frac)
    FNs[ns] = [fp_tr, fp_va, fp_te]
    with open(os.path.join(path, 'filename_{}_tr'.format(ns)), 'w') as f:
        for i in fp_tr:
            f.writelines(i+'\n')
    with open(os.path.join(path, 'filename_{}_va'.format(ns)), 'w') as f:
        for i in fp_va:
            f.writelines(i+'\n')
    with open(os.path.join(path, 'filename_{}_te'.format(ns)), 'w') as f:
        for i in fp_te:
            f.writelines(i+'\n')
fn_train = np.hstack([FNs[i][0] for i in ['lens', 'nlens']])
nl1 = np.hstack([FNs[i][0] for i in ['lens']]).shape[0]
nl0 = np.hstack([FNs[i][0] for i in ['nlens']]).shape[0]
print(nl1, nl0)
data_tr = np.empty([len(fn_train)], dtype=dt)
data_tr['label'][:nl1] = 1
data_tr['label'][nl1:nl1+nl0] = 0
for i in tqdm.tqdm(range(len(fn_train))):
    data_tr['image'][i] = readdata(os.path.join(DirBase, fn_train[i]))
fn_valid = np.hstack([FNs[i][1] for i in ['lens', 'nlens']])
nl1 = np.hstack([FNs[i][1] for i in ['lens']]).shape[0]
nl0 = np.hstack([FNs[i][1] for i in ['nlens']]).shape[0]
print(nl1, nl0)
data_va = np.empty([len(fn_valid)], dtype=dt)
data_va['label'][:nl1] = 1
data_va['label'][nl1:nl1+nl0] = 0
for i in tqdm.tqdm(range(len(fn_valid))):
    data_va['image'][i] = readdata(os.path.join(DirBase, fn_valid[i]))
fn_test = np.hstack([FNs[i][2] for i in ['lens', 'nlens']])
nl1 = np.hstack([FNs[i][2] for i in ['lens']]).shape[0]
nl0 = np.hstack([FNs[i][2] for i in ['nlens']]).shape[0]
print nl1, nl0
data_te = np.empty([len(fn_test)], dtype=dt)
data_te['label'][:nl1]=1
data_te['label'][nl1:nl1+nl0]=0
remove = []
for i in tqdm.tqdm(range(len(fn_test))):
    data_te['image'][i] = readdata(os.path.join(DirBase,fn_test[i]))<jupyter_output>1%|▏         | 29/2107 [00:00<00:08, 246.58it/s]<jupyter_text>oversampling for training and validation sets:<jupyter_code>def oversampling(sample):
    index = np.arange(sample.shape[0])
    dstar_ind = index[sample['label']==1]
    sstar_ind = index[sample['label']==0]
    resample_ind = dstar_ind[np.random.randint(0, dstar_ind.shape[0]-1, sstar_ind.shape[0])]
    ind_new = np.hstack([sstar_ind,resample_ind])
    ind_new = ind_new[np.random.permutation(np.arange(ind_new.shape[0]))]
    return ind_new
reind = oversampling(data_tr)
train_dataset = data_tr[reind]
reind = oversampling(data_va)
valid_dataset = data_va[reind]<jupyter_output><empty_output><jupyter_text>for a test case:<jupyter_code>test_dataset = data_va
np.save(os.path.join(OutBase,'data/test.npy'), test_dataset)
test_dataset = data_te
print(train_dataset.shape, valid_dataset.shape, test_dataset.shape)
np.save(os.path.join(OutBase,'data/training.npy'), train_dataset)
#np.save(os.path.join(OutBase,'data/test.npy'), test_dataset)
np.save(os.path.join(OutBase,'data/valid.npy'), valid_dataset)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18563245084378388, 'fraction_numerical': 0.03344170924291686, 'mean_word_length': 4.456081081081081, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '271592', 'license': 'MIT', 'n_tokens_mistral': 2786, 'n_tokens_neox': 2651, 'n_words': 463, 'path': 'src/create_dataset.ipynb', 'repo_name': 'POFK/LensFinder', 'size': '6459'}