<commit_before>#
# Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
import select
import sys
import time

from conary.lib import util

from rmake.build import buildjob, buildtrove
from rmake.lib.apiutils import thaw, freeze
from rmake.lib import auth, localrpc
from rmake import subscribers
from rmake.subscribers import xmlrpc


def monitorJob(client, jobId, uri, showTroveLogs=False, showBuildLogs=False):
    receiver = XMLRPCJobLogReceiver(uri, client, showTroveLogs=showTroveLogs, 
                                    showBuildLogs=showBuildLogs)
    receiver.subscribe(jobId)
    receiver.serve_forever()

def waitForJob(client, jobId, uri):
    receiver = XMLRPCJobLogReceiver(uri, client, displayClass=SilentDisplay)
    receiver.subscribe(jobId)
    receiver.serve_forever()

class _AbstractDisplay(xmlrpc.BasicXMLRPCStatusSubscriber):
    def __init__(self, client, showBuildLogs=True, out=None):
        self.client = client
        self.finished = False
        self.showBuildLogs = showBuildLogs
        if not out:
            out = sys.stdout
        self.out = out

    def _msg(self, msg, *args):
        self.out.write('[%s] %s\n' % (time.strftime('%X'), msg))
        self.out.flush()

    def _jobStateUpdated(self, jobId, state, status):
        isFinished = (state in (buildjob.JOB_STATE_FAILED,
                                buildjob.JOB_STATE_BUILT))
        if isFinished:
            self._setFinished()

    def _setFinished(self):
        self.finished = True

    def _isFinished(self):
        return self.finished

    def _primeOutput(self, client, jobId):
        job = client.getJob(jobId, withTroves=False)
        if job.isFinished():
            self._setFinished()

class SilentDisplay(_AbstractDisplay):
    def _updateBuildLog(self):
        pass

class JobLogDisplay(_AbstractDisplay):

    def __init__(self, client, showBuildLogs=True, out=None):
        _AbstractDisplay.__init__(self, client, out)
        self.showBuildLogs = showBuildLogs
        self.buildingTrove = None

    def _tailBuildLog(self, jobId, troveTuple):
        self.buildingTrove = (jobId, troveTuple, 0)
        self.out.write('Tailing %s build log:\n\n' % troveTuple[0])

    def _updateBuildLog(self):
        if not self.buildingTrove:
            return

        jobId, troveTuple, mark = self.buildingTrove

        moreData = True

        moreData, data = self.client.getTroveBuildLog(jobId, troveTuple, mark)
        mark += len(data)
        self.out.write(data)
        if not moreData:
            self.buildingTrove = None
        else:
            self.buildingTrove = (jobId, troveTuple, mark)

    def _jobTrovesSet(self, jobId, troveData):
        self._msg('[%d] - job troves set' % jobId)

    def _jobStateUpdated(self, jobId, state, status):
        _AbstractDisplay._jobStateUpdated(self, jobId, state, status)
        state = buildjob._getStateName(state)
        self._msg('[%d] - State: %s' % (jobId, state))
        if status:
            self._msg('[%d] - %s' % (jobId, status))

    def _jobLogUpdated(self, jobId, state, status):
        self._msg('[%d] %s' % (jobId, status))

    def _troveStateUpdated(self, (jobId, troveTuple), state, status):
        isBuilding = (state == buildtrove.TROVE_STATE_BUILDING)
        state = buildtrove._getStateName(state)
        self._msg('[%d] %s - State: %s' % (jobId, troveTuple[0], state))
        if status:
            self._msg('[%d] %s - %s' % (jobId, troveTuple[0], status))
        if isBuilding and self.showBuildLogs:
            self._tailBuildLog(jobId, troveTuple)

    def _troveLogUpdated(self, (jobId, troveTuple), state, status):
        state = buildtrove._getStateName(state)
        self._msg('[%d] - %s - %s' % (jobId, troveTuple[0], status))

    def _primeOutput(self, client, jobId):
        logMark = 0
        while True:
            newLogs = client.getJobLogs(jobId, logMark)
            if not newLogs:
                break
            logMark += len(newLogs)<commit_msg>naive support for tailing multiple build logs at once in monitor<commit_after>#
# Copyright (c) 2006 rPath, Inc.
#
# This program is distributed under the terms of the Common Public License,
# version 1.0. A copy of this license should have been distributed with this
# source file in a file called LICENSE. If it is not present, the license
# is always available at http://www.opensource.org/licenses/cpl.php.
#
# This program is distributed in the hope that it will be useful, but
# without any warranty; without even the implied warranty of merchantability
# or fitness for a particular purpose. See the Common Public License for
# full details.
#
import select
import sys
import time

from conary.lib import util

from rmake.build import buildjob, buildtrove
from rmake.lib.apiutils import thaw, freeze
from rmake.lib import auth, localrpc
from rmake import subscribers
from rmake.subscribers import xmlrpc


def monitorJob(client, jobId, uri, showTroveLogs=False, showBuildLogs=False):
    receiver = XMLRPCJobLogReceiver(uri, client, showTroveLogs=showTroveLogs, 
                                    showBuildLogs=showBuildLogs)
    receiver.subscribe(jobId)
    receiver.serve_forever()

def waitForJob(client, jobId, uri):
    receiver = XMLRPCJobLogReceiver(uri, client, displayClass=SilentDisplay)
    receiver.subscribe(jobId)
    receiver.serve_forever()

class _AbstractDisplay(xmlrpc.BasicXMLRPCStatusSubscriber):
    def __init__(self, client, showBuildLogs=True, out=None):
        self.client = client
        self.finished = False
        self.showBuildLogs = showBuildLogs
        if not out:
            out = sys.stdout
        self.out = out

    def _msg(self, msg, *args):
        self.out.write('[%s] %s\n' % (time.strftime('%X'), msg))
        self.out.flush()

    def _jobStateUpdated(self, jobId, state, status):
        isFinished = (state in (buildjob.JOB_STATE_FAILED,
                                buildjob.JOB_STATE_BUILT))
        if isFinished:
            self._setFinished()

    def _setFinished(self):
        self.finished = True

    def _isFinished(self):
        return self.finished

    def _primeOutput(self, client, jobId):
        job = client.getJob(jobId, withTroves=False)
        if job.isFinished():
            self._setFinished()

class SilentDisplay(_AbstractDisplay):
    def _updateBuildLog(self):
        pass

class JobLogDisplay(_AbstractDisplay):

    def __init__(self, client, showBuildLogs=True, out=None):
        _AbstractDisplay.__init__(self, client, out)
        self.showBuildLogs = showBuildLogs
        self.buildingTroves = {}

    def _tailBuildLog(self, jobId, troveTuple):
        self.buildingTroves[jobId, troveTuple] =  0
        self.out.write('Tailing %s build log:\n\n' % troveTuple[0])

    def _updateBuildLog(self):
        if not self.buildingTroves:
            return
        for (jobId, troveTuple), mark in self.buildingTroves.items():
            moreData = True
            moreData, data = self.client.getTroveBuildLog(jobId, troveTuple,
                                                          mark)
            mark += len(data)
            self.out.write(data)
            if not moreData:
                del self.buildingTroves[jobId, troveTuple]
            else:
                self.buildingTroves[jobId, troveTuple] =  mark

    def _jobTrovesSet(self, jobId, troveData):
        self._msg('[%d] - job troves set' % jobId)

    def _jobStateUpdated(self, jobId, state, status):
        _AbstractDisplay._jobStateUpdated(self, jobId, state, status)
        state = buildjob._getStateName(state)
        if self._isFinished():
            self._updateBuildLog()
        self._msg('[%d] - State: %s' % (jobId, state))
        if status:
            self._msg('[%d] - %s' % (jobId, status))

    def _jobLogUpdated(self, jobId, state, status):
        self._msg('[%d] %s' % (jobId, status))

    def _troveStateUpdated(self, (jobId, troveTuple), state, status):
        isBuilding = (state == buildtrove.TROVE_STATE_BUILDING)
        state = buildtrove._getStateName(state)
        self._msg('[%d] %s - State: %s' % (jobId, troveTuple[0], state))
        if status:
            self._msg('[%d] %s - %s' % (jobId, troveTuple[0], status))
        if isBuilding and self.showBuildLogs:
            self._tailBuildLog(jobId, troveTuple)

    def _troveLogUpdated(self, (jobId, troveTuple), state, status):
        state = buildtrove._getStateName(state)
        self._msg('[%d] - %s - %s' % (jobId, troveTuple[0], status))

    def _primeOutput(self, client, jobId):
        logMark = 0
        while True:
            newLogs = client.getJobLogs(jobId, logMark)
            if not newLogs:
                break
            logMark += len(newLogs)
{'fraction_non_alphanumeric': 0.10177182368193605, 'fraction_numerical': 0.0025929127052722557, 'mean_word_length': 2.719164323021294, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1970087', 'n_tokens_mistral': 2907, 'n_tokens_neox': 2825, 'n_words': 748}