<commit_before>from Queue import Queue
import os
import subprocess
import threading

def primitive(i, o, n, a, s, m):
    args = (i, o, n, a, s, m)
    cmd = 'primitive -i %s -o %s -n %d -a %d -s %d -m %d' % args
    subprocess.call(cmd, shell=True)

def create_jobs(in_folder, out_folder, n, a, s, m):
    result = []
    try:
        os.makedirs(out_folder)
    except Exception:
        pass
    for name in os.listdir(in_folder):
        if not name.endswith('.jpg'):
            continue
        out_name = '%s.%d.%d.%d.%d.png' % (name[:-4], n, a, s, m)
        in_path = os.path.join(in_folder, name)
        out_path = os.path.join(out_folder, out_name)
        if os.path.exists(out_path):
            continue
        result.append((in_path, out_path, n, a, s, m))
    return result

def worker(jobs, done):
    while True:
        job = jobs.get()
        log(job)
        primitive(*job)
        done.put(True)

def process(in_folder, out_folder, nlist, alist, slist, mlist, nworkers):
    jobs = Queue()
    done = Queue()
    for i in range(nworkers):
        t = threading.Thread(target=worker, args=(jobs, done))
        t.setDaemon(True)
        t.start()
    count = 0
    for n, a, s, m in itertools.product(nlist, alist, slist, mlist):
        for job in create_jobs(in_folder, out_folder, n, a, s, m):
            jobs.put(job)
            count += 1
    for i in range(count):
        done.get()

log_lock = threading.Lock()

def log(x):
    with log_lock:
        print x

if __name__ == '__main__':
    nlist = [50, 100, 150, 200]
    alist = [128]
    slist = [4]
    mlist = [0, 1, 3, 5, 2, 4]
    nworkers = 4
    process('input', 'output', nlist, alist, slist, mlist, nworkers)<commit_msg>process.py tweaks<commit_after>from Queue import Queue
import itertools
import os
import subprocess
import threading

def primitive(i, o, n, a, s, m):
    args = (i, o, n, a, s, m)
    cmd = 'primitive -i %s -o %s -n %d -a %d -s %d -m %d' % args
    subprocess.call(cmd, shell=True)

def create_jobs(in_folder, out_folder, n, a, s, m):
    result = []
    try:
        os.makedirs(out_folder)
    except Exception:
        pass
    for name in os.listdir(in_folder):
        if not name.endswith('.jpg'):
            continue
        out_name = '%s.%d.%d.%d.%d.png' % (name[:-4], n, a, s, m)
        in_path = os.path.join(in_folder, name)
        out_path = os.path.join(out_folder, out_name)
        if os.path.exists(out_path):
            continue
        result.append((in_path, out_path, n, a, s, m))
    return result

def worker(jobs, done):
    while True:
        job = jobs.get()
        log(job)
        primitive(*job)
        done.put(True)

def process(in_folder, out_folder, nlist, alist, slist, mlist, nworkers):
    jobs = Queue()
    done = Queue()
    for i in xrange(nworkers):
        t = threading.Thread(target=worker, args=(jobs, done))
        t.setDaemon(True)
        t.start()
    count = 0
    items = []
    for n, a, s, m in itertools.product(nlist, alist, slist, mlist):
        for i, job in enumerate(create_jobs(in_folder, out_folder, n, a, s, m)):
            key = (n, i, m)
            items.append((key, job))
    items.sort()
    for _, job in items:
        jobs.put(job)
        count += 1
    for i in xrange(count):
        done.get()

log_lock = threading.Lock()

def log(x):
    with log_lock:
        print x

if __name__ == '__main__':
    nlist = [50, 100, 200]
    alist = [128]
    slist = [4]
    mlist = [1, 3, 5]
    nworkers = 4
    process('input1', 'output1', nlist, alist, slist, mlist, nworkers)
{'fraction_non_alphanumeric': 0.1425371044525343, 'fraction_numerical': 0.01288154578549426, 'mean_word_length': 2.259124087591241, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4460974', 'n_tokens_mistral': 1370, 'n_tokens_neox': 1320, 'n_words': 377}