<commit_before>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from common import chromium_utils
import json
import os
import re
import tempfile


# These labels should match the ones output by gtest's JSON.
TEST_UNKNOWN_LABEL = 'UNKNOWN'
TEST_SUCCESS_LABEL = 'SUCCESS'
TEST_FAILURE_LABEL = 'FAILURE'
TEST_FAILURE_ON_EXIT_LABEL = 'FAILURE_ON_EXIT'
TEST_CRASH_LABEL = 'CRASH'
TEST_TIMEOUT_LABEL = 'TIMEOUT'
TEST_SKIPPED_LABEL = 'SKIPPED'
TEST_WARNING_LABEL = 'WARNING'

FULL_RESULTS_FILENAME = 'full_results.json'
TIMES_MS_FILENAME = 'times_ms.json'

def CompressList(lines, max_length, middle_replacement):
  """Ensures that |lines| is no longer than |max_length|. If |lines| need to
  be compressed then the middle items are replaced by |middle_replacement|.
  """
  if len(lines) <= max_length:
    return lines
  remove_from_start = max_length / 2
  return (lines[:remove_from_start] +
          [middle_replacement] +
          lines[len(lines) - (max_length - remove_from_start):])


class GTestLogParser(object):
  """This helper class process GTest test output."""

  def __init__(self):
    # State tracking for log parsing
    self.completed = False
    self._current_test = ''
    self._failure_description = []
    self._current_report_hash = ''
    self._current_report = []
    self._parsing_failures = False

    # Line number currently being processed.
    self._line_number = 0

    # List of parsing errors, as human-readable strings.
    self._internal_error_lines = []

    # Tests are stored here as 'test.name': (status, [description]).
    # The status should be one of ('started', 'OK', 'failed', 'timeout',
    # 'warning'). Warning indicates that a test did not pass when run in
    # parallel with other tests but passed when run alone. The description is
    # a list of lines detailing the test's error, as reported in the log.
    self._test_status = {}

    # Reports are stored here as 'hash': [report].
    self._memory_tool_reports = {}

    # This may be either text or a number. It will be used in the phrase
    # '%s disabled' or '%s flaky' on the waterfall display.
    self._disabled_tests = 0
    self._flaky_tests = 0

    # Regular expressions for parsing GTest logs. Test names look like
    #   SomeTestCase.SomeTest
    #   SomeName/SomeTestCase.SomeTest/1
    # This regexp also matches SomeName.SomeTest/1, which should be harmless.
    test_name_regexp = r'((\w+/)?\w+\.\w+(/\d+)?)'

    self._master_name_re = re.compile(r'\[Running for master: "([^"]*)"')
    self.master_name = ''

    self._test_name = re.compile(test_name_regexp)
    self._test_start = re.compile(r'\[\s+RUN\s+\] ' + test_name_regexp)<commit_msg>Update GTestLogParser's test name regex to allow an arbitrary number of slash-separated words<commit_after>#!/usr/bin/env python
# Copyright (c) 2012 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from common import chromium_utils
import json
import os
import re
import tempfile


# These labels should match the ones output by gtest's JSON.
TEST_UNKNOWN_LABEL = 'UNKNOWN'
TEST_SUCCESS_LABEL = 'SUCCESS'
TEST_FAILURE_LABEL = 'FAILURE'
TEST_FAILURE_ON_EXIT_LABEL = 'FAILURE_ON_EXIT'
TEST_CRASH_LABEL = 'CRASH'
TEST_TIMEOUT_LABEL = 'TIMEOUT'
TEST_SKIPPED_LABEL = 'SKIPPED'
TEST_WARNING_LABEL = 'WARNING'

FULL_RESULTS_FILENAME = 'full_results.json'
TIMES_MS_FILENAME = 'times_ms.json'

def CompressList(lines, max_length, middle_replacement):
  """Ensures that |lines| is no longer than |max_length|. If |lines| need to
  be compressed then the middle items are replaced by |middle_replacement|.
  """
  if len(lines) <= max_length:
    return lines
  remove_from_start = max_length / 2
  return (lines[:remove_from_start] +
          [middle_replacement] +
          lines[len(lines) - (max_length - remove_from_start):])


class GTestLogParser(object):
  """This helper class process GTest test output."""

  def __init__(self):
    # State tracking for log parsing
    self.completed = False
    self._current_test = ''
    self._failure_description = []
    self._current_report_hash = ''
    self._current_report = []
    self._parsing_failures = False

    # Line number currently being processed.
    self._line_number = 0

    # List of parsing errors, as human-readable strings.
    self._internal_error_lines = []

    # Tests are stored here as 'test.name': (status, [description]).
    # The status should be one of ('started', 'OK', 'failed', 'timeout',
    # 'warning'). Warning indicates that a test did not pass when run in
    # parallel with other tests but passed when run alone. The description is
    # a list of lines detailing the test's error, as reported in the log.
    self._test_status = {}

    # Reports are stored here as 'hash': [report].
    self._memory_tool_reports = {}

    # This may be either text or a number. It will be used in the phrase
    # '%s disabled' or '%s flaky' on the waterfall display.
    self._disabled_tests = 0
    self._flaky_tests = 0

    # Regular expressions for parsing GTest logs. Test names look like
    # "x.y", with 0 or more "w/" prefixes and 0 or more "/z" suffixes.
    # e.g.:
    #   SomeName/SomeTestCase.SomeTest/1
    #   SomeName/SomeTestCase/1.SomeTest
    #   SomeName/SomeTestCase/1.SomeTest/SomeModifider
    test_name_regexp = r'((\w+/)*\w+\.\w+(/\w+)*)'

    self._master_name_re = re.compile(r'\[Running for master: "([^"]*)"')
    self.master_name = ''

    self._test_name = re.compile(test_name_regexp)
    self._test_start = re.compile(r'\[\s+RUN\s+\] ' + test_name_regexp)
{'fraction_non_alphanumeric': 0.10369342812554187, 'fraction_numerical': 0.003988208774059303, 'mean_word_length': 3.685621445978879, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4096267', 'n_tokens_mistral': 1845, 'n_tokens_neox': 1790, 'n_words': 569}