<commit_before>import logging

from django.db import (IntegrityError,
                       transaction)
from mozlog.formatters.tbplformatter import TbplFormatter

from treeherder.model.models import (FailureLine,
                                     Job,
                                     TextLogError,
                                     TextLogErrorMetadata)

logger = logging.getLogger(__name__)


def crossreference_job(job):
    """Try to match the unstructured error lines with the corresponding structured error
    lines, relying on the fact that serialization of mozlog (and hence errorsummary files)
    is determinisic so we can reserialize each structured error line and perform an in-order
    textual match.

    :job: - Job for which to perform the crossreferencing
    """

    if job.autoclassify_status >= Job.CROSSREFERENCED:
        logger.info("Job %i already crossreferenced" % job.id)
        return False
    try:
        rv = _crossreference(job)
    except IntegrityError:
        job.autoclassify_status = Job.FAILED
        job.save(update_fields=['autoclassify_status'])
        logger.warning("IntegrityError crossreferencing error lines for job %s" % job.id)
        return False

    job.autoclassify_status = Job.CROSSREFERENCED
    job.save(update_fields=['autoclassify_status'])
    return rv


@transaction.atomic
def _crossreference(job):
    failure_lines = FailureLine.objects.filter(job_guid=job.guid)

    text_log_errors = TextLogError.objects.filter(
        step__job=job).order_by('line_number')

    # If we don't have both failure lines and text log errors this will never succeed
    if not (failure_lines.exists() and text_log_errors.exists()):
        return False

    match_iter = structured_iterator(list(failure_lines.all()))
    failure_line, _, fn = match_iter.next()

    # For each error in the text log, try to match the next unmatched
    # structured log line
    for error in list(text_log_errors.all()):
        if fn and fn(error.line.strip()):
            logger.debug("Matched '%s'" % (error.line,))
            TextLogErrorMetadata.objects.get_or_create(text_log_error=error,
                                                       failure_line=failure_line)
            failure_line, _, fn = match_iter.next()
        else:
            logger.debug("Failed to match '%s'" % (error.line,))

    # We should have exhausted all structured lines
    for failure_line, repr_str, _ in match_iter:
        # We can have a line without a pattern at the end if the log is truncated
        if failure_line is None:
            break
        logger.warning("Crossreference %s: Failed to match structured line '%s' to an unstructured line" %
                       (job.id, repr_str))

    return True


def structured_iterator(failure_lines):
    """Map failure_lines to a (failure_line, regexp) iterator where the
    regexp will match an unstructured line corresponding to that structured line.

    :param failure_lines: Iterator of FailureLine objects
    """
    to_fn = ErrorSummaryMatchConvertor()
    for failure_line in failure_lines:
        repr_str, fn = to_fn(failure_line)
        if fn:
            yield failure_line, repr_str, fn
    while True:
        yield None, None, None
<commit_msg>Bug 1451518 - Use next(obj) instead of obj.next()<commit_after>import logging

from django.db import (IntegrityError,
                       transaction)
from mozlog.formatters.tbplformatter import TbplFormatter

from treeherder.model.models import (FailureLine,
                                     Job,
                                     TextLogError,
                                     TextLogErrorMetadata)

logger = logging.getLogger(__name__)


def crossreference_job(job):
    """Try to match the unstructured error lines with the corresponding structured error
    lines, relying on the fact that serialization of mozlog (and hence errorsummary files)
    is determinisic so we can reserialize each structured error line and perform an in-order
    textual match.

    :job: - Job for which to perform the crossreferencing
    """

    if job.autoclassify_status >= Job.CROSSREFERENCED:
        logger.info("Job %i already crossreferenced" % job.id)
        return False
    try:
        rv = _crossreference(job)
    except IntegrityError:
        job.autoclassify_status = Job.FAILED
        job.save(update_fields=['autoclassify_status'])
        logger.warning("IntegrityError crossreferencing error lines for job %s" % job.id)
        return False

    job.autoclassify_status = Job.CROSSREFERENCED
    job.save(update_fields=['autoclassify_status'])
    return rv


@transaction.atomic
def _crossreference(job):
    failure_lines = FailureLine.objects.filter(job_guid=job.guid)

    text_log_errors = TextLogError.objects.filter(
        step__job=job).order_by('line_number')

    # If we don't have both failure lines and text log errors this will never succeed
    if not (failure_lines.exists() and text_log_errors.exists()):
        return False

    match_iter = structured_iterator(list(failure_lines.all()))
    failure_line, _, fn = next(match_iter)

    # For each error in the text log, try to match the next unmatched
    # structured log line
    for error in list(text_log_errors.all()):
        if fn and fn(error.line.strip()):
            logger.debug("Matched '%s'" % (error.line,))
            TextLogErrorMetadata.objects.get_or_create(text_log_error=error,
                                                       failure_line=failure_line)
            failure_line, _, fn = next(match_iter)
        else:
            logger.debug("Failed to match '%s'" % (error.line,))

    # We should have exhausted all structured lines
    for failure_line, repr_str, _ in match_iter:
        # We can have a line without a pattern at the end if the log is truncated
        if failure_line is None:
            break
        logger.warning("Crossreference %s: Failed to match structured line '%s' to an unstructured line" %
                       (job.id, repr_str))

    return True


def structured_iterator(failure_lines):
    """Map failure_lines to a (failure_line, regexp) iterator where the
    regexp will match an unstructured line corresponding to that structured line.

    :param failure_lines: Iterator of FailureLine objects
    """
    to_fn = ErrorSummaryMatchConvertor()
    for failure_line in failure_lines:
        repr_str, fn = to_fn(failure_line)
        if fn:
            yield failure_line, repr_str, fn
    while True:
        yield None, None, None

{'fraction_non_alphanumeric': 0.07334144856968959, 'fraction_numerical': 0.0010651247717589774, 'mean_word_length': 2.591803278688525, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6526821', 'n_tokens_mistral': 1826, 'n_tokens_neox': 1753, 'n_words': 556}