<commit_before># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Impact analysis."""

import collections
import datetime
import logging
import os
import subprocess
import tempfile

from google.cloud import ndb
import pygit2

from . import ecosystems
from . import repos
from . import models
from . import vulnerability_pb2

COMMIT_RANGE_LIMIT = 4

TAG_PREFIX = 'refs/tags/'

# Used in cases where an earlier commit in a regression range cannot be
# determined.
UNKNOWN_COMMIT = 'unknown'

AffectedResult = collections.namedtuple(
    'AffectedResult', 'tags_with_bug tags_with_fix commits affected_ranges '
    'regress_commits fix_commits')

AnalyzeResult = collections.namedtuple('AnalyzeResult', 'has_changes commits')

TagsInfo = collections.namedtuple('TagsInfo', 'tags latest_tag')


class ImpactError(Exception):
  """Impact error."""


class RangeCollector:
  """Affected range collector (preserves insertion order)."""

  def __init__(self):
    self.grouped_ranges = {}

  def add(self, introduced_in, fixed_in):
    """Add a new commit range."""
    if introduced_in in self.grouped_ranges:
      if fixed_in is None:
        # New range doesn't add anything new.
        return

      # Remove in-place as we need to preserve insertion order.
      ranges = self.grouped_ranges[introduced_in]
      ranges.append((introduced_in, fixed_in))
      for value in ranges.copy():
        if value[1] is None:
          ranges.remove(value)
    else:
      self.grouped_ranges[introduced_in] = [(introduced_in, fixed_in)]

  def ranges(self):
    """Return a set representing the collected commit ranges."""
    ranges = []
    for grouped_range in self.grouped_ranges.values():
      for commit_range in grouped_range:
        if commit_range not in ranges:
          ranges.append(commit_range)

    return ranges


class RepoAnalyzer:
  """Repository analyzer."""

  def __init__(self, detect_cherrypicks=True):
    self.detect_cherrypicks = detect_cherrypicks

  def get_affected(self, repo, regress_commit_or_range, fix_commit_or_range):
    """"Get list of affected tags and commits for a bug given regressed and
    fixed commits."""
    regress_commits = get_commit_range(repo, regress_commit_or_range)
    if len(regress_commits) > COMMIT_RANGE_LIMIT:
      raise ImpactError('Too many commits in regression range.')

    # If multiple, assume the first commit in the regression range cause the
    # regression.
    if regress_commits:
      regress_commit = regress_commits[0]
    else:
      regress_commit = None

    fix_commits = get_commit_range(repo, fix_commit_or_range)
    if len(fix_commits) > COMMIT_RANGE_LIMIT:
      logging.warning('Too many commits in fix range.')
      # Rather than bail out here and potentially leaving a Bug as "unfixed"
      # indefinitely, we continue.

    # If multiple, assume the last commit is necessary for fixing the
    # regression.
    if fix_commits:
      fix_commit = fix_commits[-1]
    else:
      fix_commit = None

    tags_with_bug = set()
    tags_with_bug.update(self.get_tags_with_commit(repo, regress_commit))

    if not regress_commits:
      # If no introduced commit provided, assume all commits prior to fix are
      # vulnerable.
      tags_with_bug.update(get_all_tags(repo))

    tags_with_fix = self.get_tags_with_commit(repo, fix_commit)
    affected_commits, affected_ranges = self.get_affected_range(
        repo, regress_commit, fix_commit)

    if len(regress_commits) > 1 or len(fix_commits) > 1:
      # Don't return ranges if input regressed and fixed commits are not single
      # commits.
      affected_ranges = []

    return AffectedResult(tags_with_bug, tags_with_fix, affected_commits,
                          affected_ranges, regress_commits, fix_commits)

  def get_affected_range(self, repo, regress_commit, fix_commit):
    """Get affected range."""
    range_collector = RangeCollector()
    commits = set()
    seen_commits = set()

    if self.detect_cherrypicks:
      # Check all branches for cherry picked regress/fix commits (sorted for
      # determinism).
      branches = sorted(repo.branches.remote)
    else:
      branches = [repo.head.name.replace('refs/heads/', 'origin/')]

    for branch in branches:
      ref = 'refs/remotes/' + branch

      # Get the earliest equivalent commit in the regression range.
      equivalent_regress_commit = None
      if self.detect_cherrypicks:
        logging.info('Finding equivalent regress commit to %s in %s',
                     regress_commit, ref)
        equivalent_regress_commit = self.get_equivalent_commit(
            repo, ref, regress_commit)
      else:
        if regress_commit:
          equivalent_regress_commit = regress_commit

      # If regress_commits is provided, then we should find an equivalent.
      if not equivalent_regress_commit and regress_commit:
        continue

      # Get the latest equivalent commit in the fix range.
      equivalent_fix_commit = None
      if self.detect_cherrypicks:
        logging.info('Finding equivalent fix commit to %s in %s', fix_commit,
                     ref)
        equivalent_fix_commit = self.get_equivalent_commit(
            repo, ref, fix_commit)
      else:
        equivalent_fix_commit = fix_commit

      range_collector.add(equivalent_regress_commit, equivalent_fix_commit)<commit_msg>Minor code cleanup. (#176)<commit_after># Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Impact analysis."""

import collections
import datetime
import logging
import os
import subprocess
import tempfile

from google.cloud import ndb
import pygit2

from . import ecosystems
from . import repos
from . import models
from . import vulnerability_pb2

COMMIT_RANGE_LIMIT = 4

TAG_PREFIX = 'refs/tags/'

# Used in cases where an earlier commit in a regression range cannot be
# determined.
UNKNOWN_COMMIT = 'unknown'

AffectedResult = collections.namedtuple(
    'AffectedResult', 'tags_with_bug tags_with_fix commits affected_ranges '
    'regress_commits fix_commits')

AnalyzeResult = collections.namedtuple('AnalyzeResult', 'has_changes commits')

TagsInfo = collections.namedtuple('TagsInfo', 'tags latest_tag')


class ImpactError(Exception):
  """Impact error."""


class RangeCollector:
  """Affected range collector (preserves insertion order)."""

  def __init__(self):
    self.grouped_ranges = {}

  def add(self, introduced_in, fixed_in):
    """Add a new commit range."""
    if introduced_in in self.grouped_ranges:
      if fixed_in is None:
        # New range doesn't add anything new.
        return

      # Remove in-place as we need to preserve insertion order.
      ranges = self.grouped_ranges[introduced_in]
      ranges.append((introduced_in, fixed_in))
      for value in ranges.copy():
        if value[1] is None:
          ranges.remove(value)
    else:
      self.grouped_ranges[introduced_in] = [(introduced_in, fixed_in)]

  def ranges(self):
    """Return a set representing the collected commit ranges."""
    ranges = []
    for grouped_range in self.grouped_ranges.values():
      for commit_range in grouped_range:
        if commit_range not in ranges:
          ranges.append(commit_range)

    return ranges


class RepoAnalyzer:
  """Repository analyzer."""

  def __init__(self, detect_cherrypicks=True):
    self.detect_cherrypicks = detect_cherrypicks

  def get_affected(self, repo, regress_commit_or_range, fix_commit_or_range):
    """"Get list of affected tags and commits for a bug given regressed and
    fixed commits."""
    regress_commits = get_commit_range(repo, regress_commit_or_range)
    if len(regress_commits) > COMMIT_RANGE_LIMIT:
      raise ImpactError('Too many commits in regression range.')

    # If multiple, assume the first commit in the regression range cause the
    # regression.
    if regress_commits:
      regress_commit = regress_commits[0]
    else:
      regress_commit = None

    fix_commits = get_commit_range(repo, fix_commit_or_range)
    if len(fix_commits) > COMMIT_RANGE_LIMIT:
      logging.warning('Too many commits in fix range.')
      # Rather than bail out here and potentially leaving a Bug as "unfixed"
      # indefinitely, we continue.

    # If multiple, assume the last commit is necessary for fixing the
    # regression.
    if fix_commits:
      fix_commit = fix_commits[-1]
    else:
      fix_commit = None

    tags_with_bug = set()
    tags_with_bug.update(self.get_tags_with_commit(repo, regress_commit))

    if not regress_commits:
      # If no introduced commit provided, assume all commits prior to fix are
      # vulnerable.
      tags_with_bug.update(get_all_tags(repo))

    tags_with_fix = self.get_tags_with_commit(repo, fix_commit)
    affected_commits, affected_ranges = self.get_affected_range(
        repo, regress_commit, fix_commit)

    if len(regress_commits) > 1 or len(fix_commits) > 1:
      # Don't return ranges if input regressed and fixed commits are not single
      # commits.
      affected_ranges = []

    return AffectedResult(tags_with_bug, tags_with_fix, affected_commits,
                          affected_ranges, regress_commits, fix_commits)

  def get_affected_range(self, repo, regress_commit, fix_commit):
    """Get affected range."""
    range_collector = RangeCollector()
    commits = set()
    seen_commits = set()

    if self.detect_cherrypicks:
      # Check all branches for cherry picked regress/fix commits (sorted for
      # determinism).
      branches = sorted(repo.branches.remote)
    else:
      branches = [repo.head.name.replace('refs/heads/', 'origin/')]

    for branch in branches:
      ref = 'refs/remotes/' + branch

      # Get the earliest equivalent commit in the regression range.
      if self.detect_cherrypicks:
        logging.info('Finding equivalent regress commit to %s in %s',
                     regress_commit, ref)
        equivalent_regress_commit = self.get_equivalent_commit(
            repo, ref, regress_commit)
      else:
        equivalent_regress_commit = regress_commit

      # If regress_commits is provided, then we should find an equivalent.
      if not equivalent_regress_commit and regress_commit:
        continue

      # Get the latest equivalent commit in the fix range.
      if self.detect_cherrypicks:
        logging.info('Finding equivalent fix commit to %s in %s', fix_commit,
                     ref)
        equivalent_fix_commit = self.get_equivalent_commit(
            repo, ref, fix_commit)
      else:
        equivalent_fix_commit = fix_commit

      range_collector.add(equivalent_regress_commit, equivalent_fix_commit)
{'fraction_non_alphanumeric': 0.07601786634598866, 'fraction_numerical': 0.0030063562961690432, 'mean_word_length': 3.3688555347091933, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2082183', 'n_tokens_mistral': 3474, 'n_tokens_neox': 3318, 'n_words': 1060}