<commit_before>from haloanalysis.batchfarm import utils,lsf
from haloanalysis.batchfarm.lsf import lsfDefaults
import haloanalysis
import yaml
import argparse
from astropy.table import Table, vstack
from os import path
import os
import logging
from glob import glob
from astropy.io import fits

if __name__ == '__main__':
    usage = "usage: %(prog)s"
    description = "Fit SED for cascade component."
    parser = argparse.ArgumentParser(usage=usage,description=description)
    parser.add_argument('-c', default = False, required=True,
                        help='yaml config file')
    args = parser.parse_args()
    utils.init_logging('INFO', color = True)
    config = yaml.load(open(args.c))
    config['configname'] = 'fit_igmf_th_jet{0[th_jet]:.0f}_tmax{0[tmax]:.0e}_lp_{0[kind]:s}'.format(config)

    tab_sed_tev = Table.read(config['cat_tev'])
    njobs = tab_sed_tev['SOURCE_FULL'].data.shape[0]

    outfile = 'fit_igmf_th_jet{0[th_jet]:.0f}_tmax{0[tmax]:.0e}_lp_{0[kind]:s}_0*.fits'.format(config)
    outfile = path.join(config['outdir'],outfile)
    print outfile
    missing = utils.missing_files(outfile, njobs, num = 4, split = '.fits')

    if len(missing) < njobs:
	njobs = missing
	logging.info('there are {0:n} files missing in {1:s}'.format(len(missing),
	    outfile ))

    if len(missing):<commit_msg>minor bug fixes<commit_after>from haloanalysis.batchfarm import utils,lsf
from haloanalysis.batchfarm.lsf import lsfDefaults
import haloanalysis
import yaml
import argparse
from astropy.table import Table, vstack
from os import path
import os
import logging
from glob import glob
from astropy.io import fits

if __name__ == '__main__':
    usage = "usage: %(prog)s"
    description = "Fit SED for cascade component."
    parser = argparse.ArgumentParser(usage=usage,description=description)
    parser.add_argument('-c', default = False, required=True,
                        help='yaml config file')
    args = parser.parse_args()
    utils.init_logging('INFO', color = True)
    config = yaml.load(open(args.c))
    config['configname'] = 'fit_igmf_th_jet{0[th_jet]:.0f}_tmax{0[tmax]:.0e}_lp_{0[kind]:s}'.format(config)

    tab_sed_tev = Table.read(config['cat_tev'])
    njobs = tab_sed_tev['SOURCE_FULL'].data.shape[0]

    outfile = 'fit_igmf_th_jet{0[th_jet]:.0f}_tmax{0[tmax]:.0e}_lp_{0[kind]:s}_0*.fits'.format(config)
    outfile = path.join(config['outdir'],outfile)
    missing = utils.missing_files(outfile, njobs, num = 4, split = '.fits')

    if len(missing) < njobs:
	njobs = missing
	logging.info('there are {0:n} files missing in {1:s}'.format(len(missing),
	    outfile ))

    if len(missing):
{'fraction_non_alphanumeric': 0.1360544217687075, 'fraction_numerical': 0.011337868480725623, 'mean_word_length': 4.480331262939958, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6181904', 'n_tokens_mistral': 948, 'n_tokens_neox': 907, 'n_words': 193}