<commit_before>import os
import tempfile
try:
    import subprocess32 as subprocess
except:
    import subprocess
import random

from concurrent.futures import ThreadPoolExecutor
import re
from math import sqrt, log1p

from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestermisc.info_utils import PandaQueuesDict


# logger
baseLogger = core_utils.setup_logger('htcondor_submitter')


# Integer division round up
def _div_round_up(a, b):
    return a // b + int(a % b > 0)


# Compute weight of each CE according to worker stat, return dict
def _get_ce_weight_dict(ce_endpoint_list=[], queue_status_dict={}, worker_ce_stats_dict={}):
    print('ce_endpoint_list', ce_endpoint_list)
    print('queue_status_dict', queue_status_dict)
    print('worker_ce_stats_dict', worker_ce_stats_dict)
    multiplier = 10.0
    n_ce = len(ce_endpoint_list)
    def _get_init_weight(_ce_endpoint):
        if ( _ce_endpoint not in worker_ce_stats_dict or n_ce == 1 ):
            return float(1)
        N = float(n_ce)
        Q = float(queue_status_dict['nQueueLimitWorker'])
        R = float(queue_status_dict['maxWorkers'])
        q = float(worker_ce_stats_dict[_ce_endpoint]['submitted'])
        r = float(worker_ce_stats_dict[_ce_endpoint]['running'])
        if ( _ce_endpoint in worker_ce_stats_dict and q > Q ):
            return float(0)<commit_msg>Remove print<commit_after>import os
import tempfile
try:
    import subprocess32 as subprocess
except:
    import subprocess
import random

from concurrent.futures import ThreadPoolExecutor
import re
from math import sqrt, log1p

from pandaharvester.harvesterconfig import harvester_config
from pandaharvester.harvestercore.queue_config_mapper import QueueConfigMapper
from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestermisc.info_utils import PandaQueuesDict


# logger
baseLogger = core_utils.setup_logger('htcondor_submitter')


# Integer division round up
def _div_round_up(a, b):
    return a // b + int(a % b > 0)


# Compute weight of each CE according to worker stat, return dict
def _get_ce_weight_dict(ce_endpoint_list=[], queue_status_dict={}, worker_ce_stats_dict={}):
    multiplier = 10.0
    n_ce = len(ce_endpoint_list)
    def _get_init_weight(_ce_endpoint):
        if ( _ce_endpoint not in worker_ce_stats_dict or n_ce == 1 ):
            return float(1)
        N = float(n_ce)
        Q = float(queue_status_dict['nQueueLimitWorker'])
        R = float(queue_status_dict['maxWorkers'])
        q = float(worker_ce_stats_dict[_ce_endpoint]['submitted'])
        r = float(worker_ce_stats_dict[_ce_endpoint]['running'])
        if ( _ce_endpoint in worker_ce_stats_dict and q > Q ):
            return float(0)
{'fraction_non_alphanumeric': 0.0756918972990997, 'fraction_numerical': 0.006668889629876626, 'mean_word_length': 4.639097744360902, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '527386', 'n_tokens_mistral': 974, 'n_tokens_neox': 930, 'n_words': 228}