<commit_before># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Grouper for grouping similar looking testcases."""

import six

from clusterfuzz._internal.base import errors
from clusterfuzz._internal.crash_analysis.crash_comparer import CrashComparer
from clusterfuzz._internal.datastore import data_handler
from clusterfuzz._internal.datastore import data_types
from clusterfuzz._internal.metrics import logs
from libs.issue_management import issue_tracker_utils

from . import group_leader

FORWARDED_ATTRIBUTES = ('crash_state', 'crash_type', 'group_id',
                        'one_time_crasher_flag', 'project_name',
                        'security_flag', 'timestamp', 'job_type')

GROUP_MAX_TESTCASE_LIMIT = 25


class TestcaseAttributes(object):
  """Testcase attributes used for grouping."""

  __slots__ = ('id', 'is_leader', 'issue_id') + FORWARDED_ATTRIBUTES

  def __init__(self, testcase_id):
    self.id = testcase_id
    self.is_leader = True
    self.issue_id = None


def combine_testcases_into_group(testcase_1, testcase_2, testcase_map):
  """Combine two testcases into a group."""
  logs.log(
      'Grouping testcase 1 '
      '(crash_type=%s, crash_state=%s, security_flag=%s, group=%s) '
      'and testcase 2 '
      '(crash_type=%s, crash_state=%s, security_flag=%s, group=%s).' %
      (testcase_1.crash_type, testcase_1.crash_state, testcase_1.security_flag,
       testcase_1.group_id, testcase_2.crash_type, testcase_2.crash_state,
       testcase_2.security_flag, testcase_2.group_id))

  # If none of the two testcases have a group id, just assign a new group id to
  # both.
  if not testcase_1.group_id and not testcase_2.group_id:
    new_group_id = _get_new_group_id()
    testcase_1.group_id = new_group_id
    testcase_2.group_id = new_group_id
    return

  # If one of the testcase has a group id, then assign the other to reuse that
  # group id.
  if testcase_1.group_id and not testcase_2.group_id:
    testcase_2.group_id = testcase_1.group_id
    return
  if testcase_2.group_id and not testcase_1.group_id:
    testcase_1.group_id = testcase_2.group_id
    return

  # If both the testcase have their own groups, then just merge the two groups
  # together and reuse one of their group ids.
  group_id_to_reuse = testcase_1.group_id
  group_id_to_move = testcase_2.group_id
  for testcase in six.itervalues(testcase_map):
    if testcase.group_id == group_id_to_move:
      testcase.group_id = group_id_to_reuse


def _get_new_group_id():
  """Get a new group id for testcase grouping."""
  new_group = data_types.TestcaseGroup()
  new_group.put()
  return new_group.key.id()


def is_same_variant(variant1, variant2):
  """Checks for the testcase variants equality."""
  return (variant1.crash_type == variant2.crash_type and
          variant1.crash_state == variant2.crash_state and
          variant1.security_flag == variant2.security_flag)


def _group_testcases_based_on_variants(testcase_map):
  """Group testcases that are associated based on variant analysis."""
  logs.log('Grouping based on variant analysis.')
  for testcase_1_id, testcase_1 in testcase_map.items():
    for testcase_2_id, testcase_2 in testcase_map.items():
      # Rule: Don't group the same testcase and use different combinations for
      # comparisons.
      if testcase_1_id <= testcase_2_id:
        continue

      # Rule: If both testcase have the same group id, then no work to do.
      if testcase_1.group_id == testcase_2.group_id and testcase_1.group_id:
        continue

      # Rule: Check both testcase are under the same project.
      if testcase_1.project_name != testcase_2.project_name:
        continue

      # Rule: If both testcase have same job_type, then skip variant anlysis.
      if testcase_1.job_type == testcase_2.job_type:
        continue

      # Rule: Group testcase with similar variants.
      # For each testcase2, get the related variant1 and check for equivalence.
      candidate_variant = data_handler.get_testcase_variant(
          testcase_1_id, testcase_2.job_type)
      if not is_same_variant(candidate_variant, testcase_2):
        continue

      # combine_testcases_into_group(testcase_1, testcase_2, testcase_map)
      # TODO(navidem): Temporary logging, should be replaced with the combine.
      logs.log('VARIANT ANALYSIS: Grouping testcase 1 '
               '(crash_type=%s, crash_state=%s, security_flag=%s, group=%s) '
               'and testcase 2 '
               '(crash_type=%s, crash_state=%s, security_flag=%s, group=%s).' %
               (testcase_1.crash_type, testcase_1.crash_state,
                testcase_1.security_flag, testcase_1.group_id,
                testcase_2.crash_type, testcase_2.crash_state,
                testcase_2.security_flag, testcase_2.group_id))


def _group_testcases_with_same_issues(testcase_map):
  """Group testcases that are associated with same underlying issue."""
  logs.log('Grouping based on same issues.')
  for testcase_1_id, testcase_1 in six.iteritems(testcase_map):
    for testcase_2_id, testcase_2 in six.iteritems(testcase_map):
      # Rule: Don't group the same testcase and use different combinations for
      # comparisons.
      if testcase_1_id <= testcase_2_id:
        continue

      # Rule: If both testcase have the same group id, then no work to do.
      if testcase_1.group_id == testcase_2.group_id and testcase_1.group_id:
        continue

      # Rule: Check both testcase have an associated issue id.
      if testcase_1.issue_id is None or testcase_2.issue_id is None:<commit_msg>More verbose logging for variant analysis (#2805)<commit_after># Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Grouper for grouping similar looking testcases."""

import six

from clusterfuzz._internal.base import errors
from clusterfuzz._internal.crash_analysis.crash_comparer import CrashComparer
from clusterfuzz._internal.datastore import data_handler
from clusterfuzz._internal.datastore import data_types
from clusterfuzz._internal.metrics import logs
from libs.issue_management import issue_tracker_utils

from . import group_leader

FORWARDED_ATTRIBUTES = ('crash_state', 'crash_type', 'group_id',
                        'one_time_crasher_flag', 'project_name',
                        'security_flag', 'timestamp', 'job_type')

GROUP_MAX_TESTCASE_LIMIT = 25


class TestcaseAttributes(object):
  """Testcase attributes used for grouping."""

  __slots__ = ('id', 'is_leader', 'issue_id') + FORWARDED_ATTRIBUTES

  def __init__(self, testcase_id):
    self.id = testcase_id
    self.is_leader = True
    self.issue_id = None


def combine_testcases_into_group(testcase_1, testcase_2, testcase_map):
  """Combine two testcases into a group."""
  logs.log(
      'Grouping testcase 1 '
      '(crash_type=%s, crash_state=%s, security_flag=%s, group=%s) '
      'and testcase 2 '
      '(crash_type=%s, crash_state=%s, security_flag=%s, group=%s).' %
      (testcase_1.crash_type, testcase_1.crash_state, testcase_1.security_flag,
       testcase_1.group_id, testcase_2.crash_type, testcase_2.crash_state,
       testcase_2.security_flag, testcase_2.group_id))

  # If none of the two testcases have a group id, just assign a new group id to
  # both.
  if not testcase_1.group_id and not testcase_2.group_id:
    new_group_id = _get_new_group_id()
    testcase_1.group_id = new_group_id
    testcase_2.group_id = new_group_id
    return

  # If one of the testcase has a group id, then assign the other to reuse that
  # group id.
  if testcase_1.group_id and not testcase_2.group_id:
    testcase_2.group_id = testcase_1.group_id
    return
  if testcase_2.group_id and not testcase_1.group_id:
    testcase_1.group_id = testcase_2.group_id
    return

  # If both the testcase have their own groups, then just merge the two groups
  # together and reuse one of their group ids.
  group_id_to_reuse = testcase_1.group_id
  group_id_to_move = testcase_2.group_id
  for testcase in six.itervalues(testcase_map):
    if testcase.group_id == group_id_to_move:
      testcase.group_id = group_id_to_reuse


def _get_new_group_id():
  """Get a new group id for testcase grouping."""
  new_group = data_types.TestcaseGroup()
  new_group.put()
  return new_group.key.id()


def is_same_variant(variant1, variant2):
  """Checks for the testcase variants equality."""
  return (variant1.crash_type == variant2.crash_type and
          variant1.crash_state == variant2.crash_state and
          variant1.security_flag == variant2.security_flag)


def _group_testcases_based_on_variants(testcase_map):
  """Group testcases that are associated based on variant analysis."""
  logs.log('Grouping based on variant analysis.')
  for testcase_1_id, testcase_1 in testcase_map.items():
    for testcase_2_id, testcase_2 in testcase_map.items():
      # Rule: Don't group the same testcase and use different combinations for
      # comparisons.
      if testcase_1_id <= testcase_2_id:
        continue

      # Rule: If both testcase have the same group id, then no work to do.
      if testcase_1.group_id == testcase_2.group_id and testcase_1.group_id:
        continue

      # Rule: Check both testcase are under the same project.
      if testcase_1.project_name != testcase_2.project_name:
        continue

      # Rule: If both testcase have same job_type, then skip variant anlysis.
      if testcase_1.job_type == testcase_2.job_type:
        continue

      # Rule: Group testcase with similar variants.
      # For each testcase2, get the related variant1 and check for equivalence.
      candidate_variant = data_handler.get_testcase_variant(
          testcase_1_id, testcase_2.job_type)
      if not is_same_variant(candidate_variant, testcase_2):
        continue

      # combine_testcases_into_group(testcase_1, testcase_2, testcase_map)
      # TODO(navidem): Temporary logging, should be replaced with the combine.
      logs.log('VARIANT ANALYSIS: Grouping testcase 1 '
               '(id=%s, '
               'crash_type=%s, crash_state=%s, security_flag=%s, group=%s) '
               'and testcase 2 (id=%s, '
               'crash_type=%s, crash_state=%s, security_flag=%s, group=%s).' %
               (testcase_1.id, testcase_1.crash_type, testcase_1.crash_state,
                testcase_1.security_flag, testcase_1.group_id, testcase_2.id,
                testcase_2.crash_type, testcase_2.crash_state,
                testcase_2.security_flag, testcase_2.group_id))


def _group_testcases_with_same_issues(testcase_map):
  """Group testcases that are associated with same underlying issue."""
  logs.log('Grouping based on same issues.')
  for testcase_1_id, testcase_1 in six.iteritems(testcase_map):
    for testcase_2_id, testcase_2 in six.iteritems(testcase_map):
      # Rule: Don't group the same testcase and use different combinations for
      # comparisons.
      if testcase_1_id <= testcase_2_id:
        continue

      # Rule: If both testcase have the same group id, then no work to do.
      if testcase_1.group_id == testcase_2.group_id and testcase_1.group_id:
        continue

      # Rule: Check both testcase have an associated issue id.
      if testcase_1.issue_id is None or testcase_2.issue_id is None:
{'fraction_non_alphanumeric': 0.07886152340564345, 'fraction_numerical': 0.01435328657641494, 'mean_word_length': 3.636294896030246, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2218771', 'n_tokens_mistral': 3964, 'n_tokens_neox': 3855, 'n_words': 1103}