<commit_before>"""
this programs takes a file containing all Python programs to run as input, and 
put these programs into a workqueue, and at every instance we make sure only 
n Python programs are running

===========================
input: 

- file containing Python programs to run
- number of programs allowed to run concurrently
- time interval of checking the number of running programs
"""

import argparse, subprocess, time

def main():
	parser = argparse.ArgumentParser()
	parser.add_argument("cmdfile", type=str, help="file containing Python programs to run")
	parser.add_argument("--num", type=int, default=20, help="number of programs allowed to run concurrently")
	parser.add_argument("--interval", type=int, default=10, help="time interval of checking the number of running programs")

	args = parser.parse_args()

	command_file = args.cmdfile
	num_of_programs_allowed = args.num
	interval = args.interval

	with open(command_file, 'r') as cmdf:
	    command_list = cmdf.read().split('\n')[1:]

	total_num_jobs = len(command_list)
	next_job_index = 0

	while next_job_index < total_num_jobs:
		time.sleep(interval)
		num_of_running_jobs = len(subprocess.check_output(['pidof', 'python']).split())
		if num_of_running_jobs < num_of_programs_allowed:
			if num_of_programs_allowed - num_of_running_jobs > total_num_jobs - next_job_index:
				run_programs(command_list, next_job_index, total_num_jobs)
				next_job_index = total_num_jobs
			else:
				run_programs(command_list, next_job_index, next_job_index + num_of_programs_allowed - num_of_running_jobs)
				next_job_index += num_of_programs_allowed - num_of_running_jobs


def run_programs(command_list, start_index, end_index):
	"""
	run programs with index [start_index, end_index - 1]
	"""
	for item in range(start_index, end_index):
		command_arg = command_list[item].split()
		if command_arg[-1] == "&":  
			command_arg = command_arg[:-1]
			print ("running command: " + str(command_arg))
			subprocess.Popen(command_arg)

	return
	

if __name__ == '__main__':
	main()
	<commit_msg>fix indent for workqueue.py<commit_after>"""
this programs takes a file containing all Python programs to run as input, and 
put these programs into a workqueue, and at every instance we make sure only 
n Python programs are running

===========================
input: 

- file containing Python programs to run
- number of programs allowed to run concurrently
- time interval of checking the number of running programs
"""

import argparse, subprocess, time

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("cmdfile", type=str, help="file containing Python programs to run")
    parser.add_argument("--num", type=int, default=20, help="number of programs allowed to run concurrently")
    parser.add_argument("--interval", type=int, default=10, help="time interval of checking the number of running programs")

    args = parser.parse_args()

    command_file = args.cmdfile
    num_of_programs_allowed = args.num
    interval = args.interval

    with open(command_file, 'r') as cmdf:
        command_list = cmdf.read().split('\n')[1:]

    total_num_jobs = len(command_list)
    next_job_index = 0

    while next_job_index < total_num_jobs:
        time.sleep(interval)
        num_of_running_jobs = len(subprocess.check_output(['pidof', 'python']).split())
        if num_of_running_jobs < num_of_programs_allowed:
            if num_of_programs_allowed - num_of_running_jobs > total_num_jobs - next_job_index:
                run_programs(command_list, next_job_index, total_num_jobs)
                next_job_index = total_num_jobs
            else:
                run_programs(command_list, next_job_index, next_job_index + num_of_programs_allowed - num_of_running_jobs)
                next_job_index += num_of_programs_allowed - num_of_running_jobs


def run_programs(command_list, start_index, end_index):
    """
    run programs with index [start_index, end_index - 1]
    """
    for item in range(start_index, end_index):
        command_arg = command_list[item].split()
        if command_arg[-1] == "&":  
            command_arg = command_arg[:-1]
            print ("running command: " + str(command_arg))
            subprocess.Popen(command_arg)

    return
    

if __name__ == '__main__':
    main()
    
{'fraction_non_alphanumeric': 0.10417633410672854, 'fraction_numerical': 0.004176334106728539, 'mean_word_length': 4.1382598331346845, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3135349', 'n_tokens_mistral': 1397, 'n_tokens_neox': 1306, 'n_words': 353}