<commit_before>import os

from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestermisc.k8s_utils import k8s_Client


# logger
baseLogger = core_utils.setup_logger('k8s_sweeper')


# sweeper for K8S
class K8sSweeper(PluginBase):
    # constructor
    def __init__(self, **kwarg):
        PluginBase.__init__(self, **kwarg)
        self.k8s_client = k8s_Client()

    # kill a worker
    def kill_worker(self, workspec):
        tmpLog = self.make_logger(baseLogger, 'workerID={0}'.format(workspec.workerID),
                                  method_name='kill_worker')

        tmpRetVal = (None, 'Nothing done')

        job_id = workspec.batchID
        try:
            self.k8s_client.delete_job(job_id)
        except Exception as _e:
            errStr = 'Failed to delete a JOB with id={0} ; {1}'.format(job_id, _e)
            tmpLog.error(errStr)
            tmpRetVal = (False, errStr)

        pod_info = self.k8s_client.get_pod_info_from_job(job_id)
        job_info = self.k8s_client.get_job_info(job_id)

        if not job_info and pod_info['status'] != 'Terminating':
            try:
                self.k8s_client.delete_pod(pod_info['name'])
            except Exception as _e:
                errStr = 'Failed to delete a POD with id={0} ; {1}'.format(pod_info['name'], _e)
                tmpLog.error(errStr)
                tmpRetVal = (False, errStr)
            else:
                tmpLog.info('Deleted a JOB & POD with id={0}'.format(job_id))
                tmpRetVal = (True, '')

        return tmpRetVal


    # cleanup for a worker
    def sweep_worker(self, workspec):
        ## Make logger
        tmpLog = self.make_logger(baseLogger, 'workerID={0}'.format(workspec.workerID),
                                  method_name='sweep_worker')

        # make sure job/pod is terminated
        return self.kill_worker(workspec)<commit_msg>Modify k8s workspec parameter setup<commit_after>import os

from pandaharvester.harvestercore import core_utils
from pandaharvester.harvestercore.plugin_base import PluginBase
from pandaharvester.harvestermisc.k8s_utils import k8s_Client


# logger
baseLogger = core_utils.setup_logger('k8s_sweeper')


# sweeper for K8S
class K8sSweeper(PluginBase):
    # constructor
    def __init__(self, **kwarg):
        PluginBase.__init__(self, **kwarg)
        self.k8s_client = k8s_Client()

    # kill a worker
    def kill_worker(self, workspec):
        tmpLog = self.make_logger(baseLogger, 'workerID={0}'.format(workspec.workerID),
                                  method_name='kill_worker')

        tmpRetVal = (None, 'Nothing done')

        job_id = workspec.batchID
        try:
            self.k8s_client.delete_job(job_id)
        except Exception as _e:
            errStr = 'Failed to delete a JOB with id={0} ; {1}'.format(job_id, _e)
            tmpLog.error(errStr)
            tmpRetVal = (False, errStr)

        pod_name = workspec.get_work_params("pod_name")
        job_info = self.k8s_client.get_job_info(job_id)

        if not job_info:
            try:
                self.k8s_client.delete_pod(pod_name)
            except Exception as _e:
                errStr = 'Failed to delete a POD with id={0} ; {1}'.format(pod_name, _e)
                tmpLog.error(errStr)
                tmpRetVal = (False, errStr)
            else:
                tmpLog.info('Deleted a JOB & POD with id={0}'.format(job_id))
                tmpRetVal = (True, '')

        return tmpRetVal


    # cleanup for a worker
    def sweep_worker(self, workspec):
        ## Make logger
        tmpLog = self.make_logger(baseLogger, 'workerID={0}'.format(workspec.workerID),
                                  method_name='sweep_worker')

        # make sure job/pod is terminated
        return self.kill_worker(workspec)
{'fraction_non_alphanumeric': 0.09667440061871617, 'fraction_numerical': 0.009280742459396751, 'mean_word_length': 2.2993197278911564, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6952005', 'n_tokens_mistral': 1263, 'n_tokens_neox': 1205, 'n_words': 254}