<commit_before>#!/usr/bin/env python

import time, sys
import multiprocessing
import random
import numpy as np

from gae_lib import PENALTY_VALUE, putJob, getNextJob


NCORES = multiprocessing.cpu_count()


class Dispatcher():
    def __init__(self, eval_func, asynch=True):
        self.eval_func = eval_func
        self.asynch = asynch
        if asynch:
            self.pool = multiprocessing.Pool(processes=NCORES)  # start worker processes
        
    def get_parameters_in(self, job):
        return job.params
    
    def create_workenv(self, job):
        return None
    #     w = str(job.jobId)
    #     if os.path.exists(w):
    #         print "[-] Workdir for job %d already exists, assuming a previous job was canceled" % job.jobId
    #         shutil.rmtree(w)
    # 
    #     os.mkdir(w)
    #     os.mkdir(os.path.join(w, 'output'))
    #     shutil.copytree(SKEL_INPUT, os.path.join(w, 'input'))
    #     shutil.copy(os.path.join(BIN_PATH, BIN), w)
    # 
    #     f = open(os.path.join(w, 'input', 'parameters.in'), 'w')
    #     f.write(get_parameters_in(job))
    #     f.close()
    
    
    def call_evaluation(self, job):
        print "[+] Running job %d" % (job.jobId)
        
        try:
            if not self.asynch:
                job.proc = self.eval_func(job.jobId, job.params)
            else:
                job.proc = self.pool.apply_async(self.eval_func, [job.jobId, job.params])
        except Exception as ex:
            print ex
            job.proc = None
        
    
    
    def gather_results(self, job):
        if not self.asynch:
            return job.proc<commit_msg>print job info before  starting<commit_after>#!/usr/bin/env python

import time, sys
import multiprocessing
import random
import numpy as np

from gae_lib import PENALTY_VALUE, putJob, getNextJob


NCORES = multiprocessing.cpu_count()


class Dispatcher():
    def __init__(self, eval_func, asynch=True):
        self.eval_func = eval_func
        self.asynch = asynch
        if asynch:
            self.pool = multiprocessing.Pool(processes=NCORES)  # start worker processes
        
    def get_parameters_in(self, job):
        return job.params
    
    def create_workenv(self, job):
        return None
    #     w = str(job.jobId)
    #     if os.path.exists(w):
    #         print "[-] Workdir for job %d already exists, assuming a previous job was canceled" % job.jobId
    #         shutil.rmtree(w)
    # 
    #     os.mkdir(w)
    #     os.mkdir(os.path.join(w, 'output'))
    #     shutil.copytree(SKEL_INPUT, os.path.join(w, 'input'))
    #     shutil.copy(os.path.join(BIN_PATH, BIN), w)
    # 
    #     f = open(os.path.join(w, 'input', 'parameters.in'), 'w')
    #     f.write(get_parameters_in(job))
    #     f.close()
    
    
    def call_evaluation(self, job):
        print "[+] Running job %d" % (job.jobId), job
        
        try:
            if not self.asynch:
                job.proc = self.eval_func(job.jobId, job.params)
            else:
                job.proc = self.pool.apply_async(self.eval_func, [job.jobId, job.params])
        except Exception as ex:
            print ex
            job.proc = None
        
    
    
    def gather_results(self, job):
        if not self.asynch:
            return job.proc
{'fraction_non_alphanumeric': 0.11327057394473125, 'fraction_numerical': 0.0, 'mean_word_length': 1.9702434625788998, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2073538', 'n_tokens_mistral': 1125, 'n_tokens_neox': 1104, 'n_words': 242}