<jupyter_start><jupyter_text>get filenames<jupyter_code>fndict = da.getfilename()
print fndict['cv'].keys()<jupyter_output>['dir', 'len', 'fname']<jupyter_text>seperate the dataset to training, valid and test<jupyter_code>fndict = da.seltrainset(fndict) 
print fndict['cv'].keys()<jupyter_output>['train', 'len_valid', 'len', 'valid', 'fname', 'len_train', 'test', 'len_test', 'dir']<jupyter_text>read data<jupyter_code>data = da.loaddata(fndict, fmt='train')
for i in data['wd'].keys():
    print data['wd'][i]['wave'].min(), data['wd'][i]['wave'].max()

print data.keys()
print data['cv'].keys()
print data['cv']['cv_0'].keys()
plt.plot(data['cv']['cv_0']['wave'], data['cv']['cv_0']['flux'], 'k-', lw=0.5)
plt.xlim([3900,9000])<jupyter_output>['wdsb2', 'wd', 'yso', 'hotstars', 'fgkm', 'cv']
['cv_2', 'cv_3', 'cv_0', 'cv_1', 'cv_6', 'cv_7', 'cv_4', 'cv_5', 'cv_8', 'cv_9']
['index', 'wave', 'label', 'flux', 'sptype']<jupyter_text>interpolating spectra and smoothing by a Gaussian kernel with $\sigma=20 nm$<jupyter_code>data_set = da.smooth(wavenew, data, sigma=200)
print data_set.keys()
print data_set['cv'].dtype
print data_set['cv'][0]['flux_norm'].shape
num = 12
fig, axes = plt.subplots(figsize=[18,6*num],nrows=num, ncols=2)
i = 0
for key in name_type.iterkeys():
    for n in xrange(num/6):
        axes[i,0].plot(data[key][key+'_%d'%(data_set[key][n]['index'])]['wave'], data[key][key+'_%d'%(data_set[key][n]['index'])]['flux'], 'k-', lw=0.8, label=key)
        axes[i,1].plot(wavenew, data_set[key]['flux_norm'][n], 'k-', lw=0.8)
        [axes[i,j].legend() for j in range(2)]
        i+=1
        if i>=num:
            break<jupyter_output><empty_output><jupyter_text>Norm<jupyter_code>da.get_musigma(data_set)<jupyter_output><empty_output><jupyter_text>random oversampling<jupyter_code>subdataset = da.oversampling(data_set['cv'], 20)<jupyter_output><empty_output><jupyter_text>save data<jupyter_code>da.save('training_cv.npy', subdataset)
da.save('training_wd.npy', data_set['wd'])<jupyter_output><empty_output><jupyter_text>check<jupyter_code>for key in name_type.keys():
    a = np.load('/nfs/P100/SDSSV_Classifiers/processed_dataset/dataset/train_%s.npy'%key)
    print key, a.dtype, a.shape, a['index']
for key in name_type.keys():
    a = np.load('/nfs/P100/SDSSV_Classifiers/processed_dataset/dataset/valid_%s.npy'%key)
    print key, a.dtype, a.shape
for key in name_type.keys():
    a = np.load('/nfs/P100/SDSSV_Classifiers/processed_dataset/dataset/test_%s.npy'%key)
    print key, a.dtype, a.shape
a = np.load('/nfs/P100/SDSSV_Classifiers/processed_dataset/dataset/train_%s.npy'%'fgkm')
print a.shape
print a.dtype
print np.sum(a['label']==-1)<jupyter_output>(6751,)
[('index', '<i4'), ('label', '<i4'), ('flux_norm', '<f4', (3000,))]
0
{'fraction_non_alphanumeric': 0.22105643994211288, 'fraction_numerical': 0.026410998552821998, 'mean_word_length': 6.877492877492878, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '192474', 'license': 'MIT', 'n_tokens_mistral': 1210, 'n_tokens_neox': 1157, 'n_words': 172, 'path': 'src/test_clean.ipynb', 'repo_name': 'POFK/starspectraclassifier', 'size': '2764'}