<commit_before>#!/usr/bin/env python2.7
from __future__ import print_function

import subprocess
import time
import sys
import signal
from collections import deque

exit_val = 0
proc_que = deque()

def run_processes_in_parallel(commands):
    global exit_val
    try:
        install_signal_handlers()
        run_parallels(commands)
        exit_val = 0
        killall_and_exit()
    except Exception as es:
        killall_and_exit()

def run_parallels(commands):
    global exit_val
    retVal = 0
    max_time = 6
    start_time = time.time()
    for command in commands:
        try:
            proc = subprocess.Popen(command, executable=command[0], shell=False)#, stdout=subprocess.PIPE, stderr=subprocess.PIPE
            #print("Started", command, proc.pid)
            #sys.stdout.flush()
            proc_que.append(proc)
        except Exception as es:
            print("failed to start", command, es.strerror, file=sys.stderr)
            sys.stdout.flush()
            exit_val = es.strerror
            raise
            
    while proc_que:
        proc = proc_que.popleft()
        status = proc.poll()
        if status is None: # None means it's still alive
            #print(proc.pid, "still alive")
            proc_que.append(proc)
            #continue
        else:
            #print(proc.pid, "just died", status)
            if status != 0:
                exit_val = status
                killall_and_exit()
        sys.stdout.flush()
        time.sleep(.5)
   
def signal_handler(signum, frame):
    global exit_val
    #print("signal", signum, frame)
    exit_val = signum
    killall_and_exit()

def killall_and_exit():
    for proc in proc_que:
        proc.kill()
    sys.exit(exit_val)
    
def install_signal_handlers():
    for sig in (signal.SIGABRT, signal.SIGFPE, signal.SIGILL, signal.SIGINT, signal.SIGSEGV, signal.SIGTERM):
        signal.signal(sig, signal_handler)<commit_msg>* Fine tuned the running of parallel processes and killing them.<commit_after>#!/usr/bin/env python2.7
from __future__ import print_function

import subprocess
import time
import sys
import os
import signal

exit_val = 0
process_list = list()

def run_processes_in_parallel(commands):
    global exit_val
    try:
        install_signal_handlers()
        run_parallels(commands)
        exit_val = 0
        killall_and_exit()
    except Exception as es:
        killall_and_exit()

def run_parallels(commands):
    global exit_val
    retVal = 0
    max_time = 6
    start_time = time.time()
    for command in commands:
        try:
            proc = subprocess.Popen(command, executable=command[0], shell=False, preexec_fn=os.setsid)#, stdout=subprocess.PIPE, stderr=subprocess.PIPE
            #print("Started", command, proc.pid)
            #sys.stdout.flush()
            process_list.append(proc)
        except Exception as es:
            print("failed to start", command, es.strerror, file=sys.stderr)
            sys.stdout.flush()
            exit_val = es.strerror
            killall_and_exit()

    active_process_list = list()
    while process_list:
        for proc in process_list:
            status = proc.poll()
            if status is None: # None means it's still alive
                #sys.stdout.write(str(proc.pid) + " still alive\n")
                #sys.stdout.flush()
                active_process_list.append(proc)
                #continue
            else:
                sys.stdout.write(str(proc.pid) + " just died " + str(status) + "\n")
                sys.stdout.flush()
                if status != 0:
                    exit_val = status
                    killall_and_exit()
        process_list[:] = active_process_list
        active_process_list[:] = []
        sys.stdout.flush()
        time.sleep(.2)
   
def signal_handler(signum, frame):
    global exit_val
    #print("signal", signum, frame)
    exit_val = signum
    killall_and_exit()

def killall_and_exit():
    for proc in process_list:
        status = proc.poll()
        if status is None: # None means it's still alive
            sys.stdout.write("killall_and_exit: "+str(proc.pid) + " gets killed\n")
            sys.stdout.flush()
            os.killpg(proc.pid, signal.SIGTERM)
            #proc.kill()
    sys.exit(exit_val)
    
def install_signal_handlers():
    for sig in (signal.SIGABRT, signal.SIGFPE, signal.SIGILL, signal.SIGINT, signal.SIGSEGV, signal.SIGTERM):
        signal.signal(sig, signal_handler)
{'fraction_non_alphanumeric': 0.09661510872001794, 'fraction_numerical': 0.004034969737726967, 'mean_word_length': 2.217015140591204, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1645729', 'n_tokens_mistral': 1379, 'n_tokens_neox': 1338, 'n_words': 320}