<commit_before>import re
import ast
import json
import logging
import traceback
from urllib2 import URLError
from datetime import datetime
from django.conf import settings
from testplanner.models import (
    Device,
    TestDefinition,
    TestDefinitionRevision
)
from testrunner.models import (
    JenkinsJob,
    JenkinsBuild,
    JenkinsBuildStatus,
    LavaJob,
    LavaJobStatus,
    LavaJobResult,
    LavaJobResultStatus,
    LavaJobTestResult,
    LavaJobTestResultUnit
)
#ToDo - get rid of lava_tool. Rewrite XML-RPC using token stored in settings
from lava_tool.authtoken import (
    AuthenticatingServerProxy, 
    KeyringAuthBackend
)

RUNNING = "RUNNING"
ERROR = "ERROR"
log = logging.getLogger('testrunner')

def fetch_jenkins_builds(jenkins_db_job, jenkins_job, jenkins_build):
    lava_server = AuthenticatingServerProxy(
        settings.LAVA_SERVER_URL,
        verbose=False,
        auth_backend=KeyringAuthBackend())

    lava_job_regexp = re.compile(settings.LAVA_JOB_ID_REGEXP)
    build = jenkins_job.get_build(jenkins_build)
    # create umbrella build in DB
    db_build = create_jenkins_build(build, jenkins_db_job)
    log.debug("master build: {0} {1}".format(db_build.number, db_build.name))
    is_matrix = False
    log.debug("fetching matrix builds")
    for run in build.get_matrix_runs():
        is_matrix = True
        db_run = create_jenkins_build(run, jenkins_db_job, False, db_build)
        if 'description' in run._data and run._data['description']:
            log.debug("Jenkins build description: {0}".format(run._data['description']))
            r = lava_job_regexp.search(run._data['description'])
            if r:
                log.debug('LAVA job ID: {0}'.format(r.group('lava_job_id')))
                get_lava_job_details(r.group('lava_job_id'), db_run, lava_server)
        else:
            log.debug("no description for build")
            for r in lava_job_regexp.finditer(run.get_console()):
                log.debug('LAVA job ID: {0}'.format(r.group('lava_job_id')))
                get_lava_job_details(r.group('lava_job_id'), db_run, lava_server)
    if not is_matrix:
        db_run = create_jenkins_build(build, jenkins_db_job, False, db_build)
        if 'description' in build._data and build._data['description']:
            log.debug("Jenkins build description: {0}".format(build._data['description']))
            r = lava_job_regexp.search(build._data['description'])
            if r:
                #print 'LAVA job ID:', r.group('lava_job_id')
                get_lava_job_details(r.group('lava_job_id'), db_run, lava_server)



def get_lava_job_details(job_id, jenkins_build, lava_server=None):
    if lava_server is None:
        lava_server = AuthenticatingServerProxy(
            settings.LAVA_SERVER_URL,
            verbose=False,
            auth_backend=KeyringAuthBackend())

    try:
        job_details = lava_server.scheduler.job_details(job_id)
        #pprint(job_details)
        db_device = None
        if 'requested_device_type_id' in job_details and job_details['requested_device_type_id']:
            db_device, created = Device.objects.get_or_create(name = job_details['requested_device_type_id'])
        else:
            # try to decode the device type from the device name
            # it is assumed that device ID ends with 2 digit number added to device type
            db_device, created = Device.objects.get_or_create(name = job_details['actual_device_id'][0:-2])
        job_status = lava_server.scheduler.job_status(job_id)
        #print 'LAVA job status:', job_status['job_status']
        lava_db_job_status, created = LavaJobStatus.objects.get_or_create(name = job_status['job_status'])
        lava_job_defaults = {
            'status': lava_db_job_status,
            'device_type': db_device<commit_msg>Allowed for multiple post-build-lava for single Jenkins build<commit_after>import re
import ast
import json
import logging
import traceback
from urllib2 import URLError
from datetime import datetime
from django.conf import settings
from testplanner.models import (
    Device,
    TestDefinition,
    TestDefinitionRevision
)
from testrunner.models import (
    JenkinsJob,
    JenkinsBuild,
    JenkinsBuildStatus,
    LavaJob,
    LavaJobStatus,
    LavaJobResult,
    LavaJobResultStatus,
    LavaJobTestResult,
    LavaJobTestResultUnit
)
#ToDo - get rid of lava_tool. Rewrite XML-RPC using token stored in settings
from lava_tool.authtoken import (
    AuthenticatingServerProxy, 
    KeyringAuthBackend
)

RUNNING = "RUNNING"
ERROR = "ERROR"
log = logging.getLogger('testrunner')

def fetch_jenkins_builds(jenkins_db_job, jenkins_job, jenkins_build):
    lava_server = AuthenticatingServerProxy(
        settings.LAVA_SERVER_URL,
        verbose=False,
        auth_backend=KeyringAuthBackend())

    lava_job_regexp = re.compile(settings.LAVA_JOB_ID_REGEXP)
    build = jenkins_job.get_build(jenkins_build)
    # create umbrella build in DB
    db_build = create_jenkins_build(build, jenkins_db_job)
    log.debug("master build: {0} {1}".format(db_build.number, db_build.name))
    is_matrix = False
    log.debug("fetching matrix builds")
    for run in build.get_matrix_runs():
        is_matrix = True
        db_run = create_jenkins_build(run, jenkins_db_job, False, db_build)
        if 'description' in run._data and run._data['description']:
            log.debug("Jenkins build description: {0}".format(run._data['description']))
            for r in lava_job_regexp.finditer(run._data['description'])
                log.debug('LAVA job ID: {0}'.format(r.group('lava_job_id')))
                get_lava_job_details(r.group('lava_job_id'), db_run, lava_server)
        else:
            log.debug("no description for build")
            for r in lava_job_regexp.finditer(run.get_console()):
                log.debug('LAVA job ID: {0}'.format(r.group('lava_job_id')))
                get_lava_job_details(r.group('lava_job_id'), db_run, lava_server)
    if not is_matrix:
        db_run = create_jenkins_build(build, jenkins_db_job, False, db_build)
        if 'description' in build._data and build._data['description']:
            log.debug("Jenkins build description: {0}".format(build._data['description']))
            for r in lava_job_regexp.finditer(build._data['description'])
                #print 'LAVA job ID:', r.group('lava_job_id')
                get_lava_job_details(r.group('lava_job_id'), db_run, lava_server)



def get_lava_job_details(job_id, jenkins_build, lava_server=None):
    if lava_server is None:
        lava_server = AuthenticatingServerProxy(
            settings.LAVA_SERVER_URL,
            verbose=False,
            auth_backend=KeyringAuthBackend())

    try:
        job_details = lava_server.scheduler.job_details(job_id)
        #pprint(job_details)
        db_device = None
        if 'requested_device_type_id' in job_details and job_details['requested_device_type_id']:
            db_device, created = Device.objects.get_or_create(name = job_details['requested_device_type_id'])
        else:
            # try to decode the device type from the device name
            # it is assumed that device ID ends with 2 digit number added to device type
            db_device, created = Device.objects.get_or_create(name = job_details['actual_device_id'][0:-2])
        job_status = lava_server.scheduler.job_status(job_id)
        #print 'LAVA job status:', job_status['job_status']
        lava_db_job_status, created = LavaJobStatus.objects.get_or_create(name = job_status['job_status'])
        lava_job_defaults = {
            'status': lava_db_job_status,
            'device_type': db_device
{'fraction_non_alphanumeric': 0.09294454569122625, 'fraction_numerical': 0.0026034886748242643, 'mean_word_length': 3.296979865771812, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1645205', 'n_tokens_mistral': 2482, 'n_tokens_neox': 2427, 'n_words': 500}