<commit_before>"""

"""

import os
import sys
import logging
import shutil
from marcc_out import write_slurm

join = os.path.join


def mkdir_quiet(dr):
    # Create output directory if needed
    import errno
    if not os.path.isdir(dr):
        try:
            os.makedirs(dr)
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise


def handle_dir(dr, start_from, global_name, base_args, exp_names, exp_qsim_args, targets, submit_fh,
               use_scavenger=False, wet=False, base_mem_gb=8, base_hours=4):
    """
    Maybe this just creates a whole series of new Makefiles with only the SUBSAMPLING_ARGS line different?
    Then maybe the
    """
    for name, ar in zip(exp_names, exp_qsim_args):
        nm = '.'.join([global_name, name])
        new_makefile_base = '.'.join(['Makefile', global_name, name])
        logging.info('  Creating new Makefile: %s' % join(dr, new_makefile_base))
        with open(join(dr, new_makefile_base), 'w') as mk_out:
            for ln in open(join(dr, 'Makefile')):
                # 2 things to do: change the args passed to qsim and change the .out target names
                if ln.startswith('SUBSAMPLING_ARGS'):
                    mk_out.write('SUBSAMPLING_ARGS=%s %s\n' % (' '.join(base_args), ' '.join(ar)))
                else:
                    mk_out.write(ln.replace('.out', '.%s.out' % nm).replace(',out', ',%s.out' % nm))
        for fulltarget in targets:
            targdir, rule = fulltarget.split('/')
            if targdir != dr:
                continue
            orig_rule = rule
            rule = rule.replace('.out', '.%s.out' % nm)
            if os.path.exists(join(dr, rule)) and os.path.exists(join(dr, rule, 'DONE')):
                logging.info('    Skipping: %s/%s because DONE exists' % (dr, rule))
                continue
            logging.info('    Adding job to make target: %s/%s' % (dr, rule))
            if start_from == 'inputalign':
                dest_dir = join(dr, rule)
                src_dir = join(dr, orig_rule)
                logging.info('    Copying from original dir %s' % src_dir)
                mkdir_quiet(dest_dir)
                assert os.path.exists(src_dir)
                assert os.path.exists(join(src_dir, 'input.sam'))
                logging.info('      Copying %s to new target dir' % (join(src_dir, 'input.sam')))
                shutil.copy(join(src_dir, 'input.sam'), dest_dir)
                assert os.path.exists(join(dest_dir, 'input.sam'))
            fn = '.' + rule + '.sh'
            write_slurm(rule, fn, dr, base_mem_gb, base_hours,
                        makefile=new_makefile_base,
                        use_scavenger=use_scavenger)
            cmd = 'pushd %s && sbatch %s && popd' % (dr, fn)
            submit_fh.write(cmd + '\n')
            if wet:
                os.system(cmd)

<commit_msg>make alignment single-threaded so it's easier to schedule many simultaneous jobs<commit_after>"""

"""

import os
import sys
import logging
import shutil
from marcc_out import write_slurm

join = os.path.join


def mkdir_quiet(dr):
    # Create output directory if needed
    import errno
    if not os.path.isdir(dr):
        try:
            os.makedirs(dr)
        except OSError as exception:
            if exception.errno != errno.EEXIST:
                raise


def handle_dir(dr, start_from, global_name, base_args, exp_names, exp_qsim_args, targets, submit_fh,
               use_scavenger=False, wet=False, base_mem_gb=6, base_hours=3):
    """
    Maybe this just creates a whole series of new Makefiles with only the SUBSAMPLING_ARGS line different?
    Then maybe the
    """
    for name, ar in zip(exp_names, exp_qsim_args):
        nm = '.'.join([global_name, name])
        new_makefile_base = '.'.join(['Makefile', global_name, name])
        logging.info('  Creating new Makefile: %s' % join(dr, new_makefile_base))
        with open(join(dr, new_makefile_base), 'w') as mk_out:
            for ln in open(join(dr, 'Makefile')):
                # 2 things to do: change the args passed to qsim and change the .out target names
                if ln.startswith('SUBSAMPLING_ARGS'):
                    mk_out.write('SUBSAMPLING_ARGS=%s %s\n' % (' '.join(base_args), ' '.join(ar)))
                elif ln.startswith('NUM_CORES='):
                    mk_out.write('NUM_CORES=1\n')
                else:
                    mk_out.write(ln.replace('.out', '.%s.out' % nm).replace(',out', ',%s.out' % nm))
        for fulltarget in targets:
            targdir, rule = fulltarget.split('/')
            if targdir != dr:
                continue
            orig_rule = rule
            rule = rule.replace('.out', '.%s.out' % nm)
            if os.path.exists(join(dr, rule)) and os.path.exists(join(dr, rule, 'DONE')):
                logging.info('    Skipping: %s/%s because DONE exists' % (dr, rule))
                continue
            logging.info('    Adding job to make target: %s/%s' % (dr, rule))
            if start_from == 'inputalign':
                dest_dir = join(dr, rule)
                src_dir = join(dr, orig_rule)
                logging.info('    Copying from original dir %s' % src_dir)
                mkdir_quiet(dest_dir)
                assert os.path.exists(src_dir)
                assert os.path.exists(join(src_dir, 'input.sam'))
                logging.info('      Copying %s to new target dir' % (join(src_dir, 'input.sam')))
                shutil.copy(join(src_dir, 'input.sam'), dest_dir)
                assert os.path.exists(join(dest_dir, 'input.sam'))
            fn = '.' + rule + '.sh'
            write_slurm(rule, fn, dr, base_mem_gb, base_hours,
                        makefile=new_makefile_base,
                        use_scavenger=use_scavenger,
                        ncores=1)
            cmd = 'pushd %s && sbatch %s && popd' % (dr, fn)
            submit_fh.write(cmd + '\n')
            if wet:
                os.system(cmd)


{'fraction_non_alphanumeric': 0.12252372232395539, 'fraction_numerical': 0.001331779590477776, 'mean_word_length': 1.9207583859990278, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2884078', 'n_tokens_mistral': 1929, 'n_tokens_neox': 1848, 'n_words': 474}