<jupyter_start><jupyter_text>Vetting the Training Set performance<jupyter_code># imports
from linetools.spectra import io as lsio<jupyter_output><empty_output><jupyter_text>Load spec<jupyter_code>spec96451 = lsio.readspec(os.getenv('DROPBOX_DIR')+'/MachineLearning/DLAs/training_96451_5000.hdf5')<jupyter_output><empty_output><jupyter_text>Check false positives<jupyter_code>#idx = 79 #-- JXP bug??
#idx = 68 #-- Lyb
#idx = 67  #-- Noisy JXP
#idx = 30 # Check JXP
#idx = 4903 # Not compelling
#idx = 4992 # JXP gave 20.25
#idx = 4927 # JXP gave 20.15
#idx = 4985 # JXP never triggered
idx = 4661 # JXP never triggered
idx = 4882 # JXP never triggered
idx = 4775 # JXP didn't trigger (looks ok)
idx = 4709 # Quite possible strong DLA (with metals)
idx = 4679
spec96451.meta['headers'][idx]['PLATE'], spec96451.meta['headers'][idx]['FIBER']
4614.*(1025.722/1215.67)
4.1104*1215.67<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13766519823788545, 'fraction_numerical': 0.11563876651982379, 'mean_word_length': 5.992307692307692, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '14482', 'license': 'MIT', 'n_tokens_mistral': 438, 'n_tokens_neox': 356, 'n_words': 80, 'path': 'docs/nb/Training_set_vetting.ipynb', 'repo_name': 'JianiDing/dla_cnn', 'size': '908'}