<commit_before>#!/usr/bin/python
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
A library to generate and store the manifests for cros builders to use.
"""

import logging
import os
import re
import time

from chromite.buildbot import manifest_version
from chromite.lib import cros_build_lib as cros_lib


def _SyncGitRepo(local_dir):
  """"Clone Given git repo
  Args:
    local_dir: location with repo that should be synced.
  """
  cros_lib.RunCommand(['git', 'remote', 'update'], cwd=local_dir)
  cros_lib.RunCommand(['git', 'rebase', 'origin/master'], cwd=local_dir)


class _LKGMCandidateInfo(manifest_version.VersionInfo):
  """Class to encapsualte the chrome os lkgm candidate info

  You can instantiate this class in two ways.
  1)using a version file, specifically chromeos_version.sh,
  which contains the version information.
  2) just passing in the 4 version components (major, minor, sp, patch and
    revision number),
  Args:
    version_string: Optional version string to parse rather than from a file
    ver_maj: major version
    ver_min: minor version
    ver_sp:  sp version
    ver_patch: patch version
    ver_revision: version revision
    version_file: version file location.
  """
  LKGM_RE = '(\d+\.\d+\.\d+\.\d+)(?:-rc(\d+))?'

  def __init__(self, version_string=None, version_file=None):
    self.ver_revision = None
    if version_string:
      match = re.search(self.LKGM_RE, version_string)
      assert match, 'LKGM did not re %s' % self.LKGM_RE
      super(_LKGMCandidateInfo, self).__init__(match.group(1),
                                               incr_type='branch')
      if match.group(2):
        self.ver_revision = int(match.group(2))

    else:
      super(_LKGMCandidateInfo, self).__init__(version_file=version_file,
                                               incr_type='branch')
    if not self.ver_revision:
      self.ver_revision = 1

  def VersionString(self):
    """returns the full version string of the lkgm candidate"""
    return '%s.%s.%s.%s-rc%s' % (self.ver_maj, self.ver_min, self.ver_sp,
                                 self.ver_patch, self.ver_revision)

  @classmethod
  def VersionCompare(cls, version_string):
    """Useful method to return a comparable version of a LKGM string."""
    lkgm = cls(version_string)
    return map(int, [lkgm.ver_maj, lkgm.ver_min, lkgm.ver_sp, lkgm.ver_patch,
                     lkgm.ver_revision])

  def IncrementVersion(self, message=None, dry_run=False):
    """Increments the version by incrementing the revision #."""
    self.ver_revision += 1
    return self.VersionString()


class LKGMManager(manifest_version.BuildSpecsManager):
  """A Class to manage lkgm candidates and their states.

  Vars:
    lkgm_subdir:  Subdirectory within manifest repo to store candidates.
  """
  # Max timeout before assuming other builders have failed.
  MAX_TIMEOUT_SECONDS = 300
  # Polling timeout for checking git repo for other build statuses.
  SLEEP_TIMEOUT = 30

  # Sub-directories for LKGM and Chrome LKGM's.
  LKGM_SUBDIR = 'LKGM-candidates'
  CHROME_PFQ_SUBDIR = 'chrome-LKGM-candidates'

  def __init__(self, source_dir, checkout_repo, manifest_repo, branch,
               build_name, build_type, clobber=False,
               dry_run=True):
    """Initialize an LKGM Manager.

    Args:
      build_type:  Type of build.  Must be either chrome or binary.
    Other args see manifest_version.BuildSpecsManager.
    """
    super(LKGMManager, self).__init__(
        source_dir=source_dir, checkout_repo=checkout_repo,
        manifest_repo=manifest_repo, branch=branch, build_name=build_name,
        incr_type='branch', clobber=clobber, dry_run=dry_run)

    self.compare_versions_fn = _LKGMCandidateInfo.VersionCompare

    assert build_type in ('chrome', 'binary')
    if build_type == 'chrome':
      self.lkgm_subdir = self.CHROME_PFQ_SUBDIR
    else:
      self.lkgm_subdir = self.LKGM_SUBDIR

  def _LoadSpecs(self, version_info):
    """Loads the specifications from the working directory.
    Args:
      version_info: Info class for version information of cros.
    """
    super(LKGMManager, self)._LoadSpecs(version_info, self.lkgm_subdir)

  def _GetLatestCandidateByVersion(self, version_info):
    """Returns the latest lkgm candidate corresponding to the version file.
    Args:
      version_info: Info class for version information of cros.
    """
    if self.all:
      matched_lkgms = filter(
          lambda ver: ver.startswith(version_info.VersionString()), self.all)
      if matched_lkgms:
        return _LKGMCandidateInfo(sorted(matched_lkgms,
                                         key=self.compare_versions_fn)[-1])

    return _LKGMCandidateInfo(version_info.VersionString())


  def _SetInFlightWithRetry(self, commit_message, retries=3):
    for index in range(retries+1):
      try:
        self._SetInFlight(commit_message)
        break
      except (manifest_version.GitCommandException,
              cros_lib.RunCommandError) as e:
        last_error = 'Failed to set build in-flight: %s' % e
        logging.error(last_error)
        logging.error('Retrying:  Retry %d/%d' %
                      (index + 1, retries + 1))
    else:
      raise manifest_version.GenerateBuildSpecException(last_error)

  def CreateNewCandidate(self, version_file, retries=3):
    """Gets the version number of the next build spec to build.
      Args:
        version_file: File to use in cros when checking for cros version.
        retries: Number of retries for updating the status
      Returns:
        next_build: a string of the next build number for the builder to consume
                    or None in case of no need to build.
      Raises:
        GenerateBuildSpecException in case of failure to generate a buildspec
    """
    try:
      version_info = self._GetCurrentVersionInfo(version_file)
      self._LoadSpecs(version_info)
      lkgm_info = self._GetLatestCandidateByVersion(version_info)

      self.current_version = self._CreateNewBuildSpec(lkgm_info)
      if self.current_version:
        logging.debug('Using build spec: %s', self.current_version)
        commit_message = 'Automatic: Start %s %s' % (self.build_name,
                                                     self.current_version)
        self._SetInFlightWithRetry(commit_message, retries)

      return self.GetLocalManifest(self.current_version)

    except (cros_lib.RunCommandError,
            manifest_version.GitCommandException) as e:
      err_msg = 'Failed to generate LKGM Candidate. error: %s' % e
      logging.error(err_msg)
      raise manifest_version.GenerateBuildSpecException(err_msg)

  def GetLatestCandidate(self, version_file, retries=0):
    """Gets the version number of the next build spec to build.
      Args:
        version_file: File to use in cros when checking for cros version.
        retries: Number of retries for updating the status
      Returns:<commit_msg>Change GetLatestCandidate retries to 5<commit_after>#!/usr/bin/python
# Copyright (c) 2011 The Chromium OS Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""
A library to generate and store the manifests for cros builders to use.
"""

import logging
import os
import re
import time

from chromite.buildbot import manifest_version
from chromite.lib import cros_build_lib as cros_lib


def _SyncGitRepo(local_dir):
  """"Clone Given git repo
  Args:
    local_dir: location with repo that should be synced.
  """
  cros_lib.RunCommand(['git', 'remote', 'update'], cwd=local_dir)
  cros_lib.RunCommand(['git', 'rebase', 'origin/master'], cwd=local_dir)


class _LKGMCandidateInfo(manifest_version.VersionInfo):
  """Class to encapsualte the chrome os lkgm candidate info

  You can instantiate this class in two ways.
  1)using a version file, specifically chromeos_version.sh,
  which contains the version information.
  2) just passing in the 4 version components (major, minor, sp, patch and
    revision number),
  Args:
    version_string: Optional version string to parse rather than from a file
    ver_maj: major version
    ver_min: minor version
    ver_sp:  sp version
    ver_patch: patch version
    ver_revision: version revision
    version_file: version file location.
  """
  LKGM_RE = '(\d+\.\d+\.\d+\.\d+)(?:-rc(\d+))?'

  def __init__(self, version_string=None, version_file=None):
    self.ver_revision = None
    if version_string:
      match = re.search(self.LKGM_RE, version_string)
      assert match, 'LKGM did not re %s' % self.LKGM_RE
      super(_LKGMCandidateInfo, self).__init__(match.group(1),
                                               incr_type='branch')
      if match.group(2):
        self.ver_revision = int(match.group(2))

    else:
      super(_LKGMCandidateInfo, self).__init__(version_file=version_file,
                                               incr_type='branch')
    if not self.ver_revision:
      self.ver_revision = 1

  def VersionString(self):
    """returns the full version string of the lkgm candidate"""
    return '%s.%s.%s.%s-rc%s' % (self.ver_maj, self.ver_min, self.ver_sp,
                                 self.ver_patch, self.ver_revision)

  @classmethod
  def VersionCompare(cls, version_string):
    """Useful method to return a comparable version of a LKGM string."""
    lkgm = cls(version_string)
    return map(int, [lkgm.ver_maj, lkgm.ver_min, lkgm.ver_sp, lkgm.ver_patch,
                     lkgm.ver_revision])

  def IncrementVersion(self, message=None, dry_run=False):
    """Increments the version by incrementing the revision #."""
    self.ver_revision += 1
    return self.VersionString()


class LKGMManager(manifest_version.BuildSpecsManager):
  """A Class to manage lkgm candidates and their states.

  Vars:
    lkgm_subdir:  Subdirectory within manifest repo to store candidates.
  """
  # Max timeout before assuming other builders have failed.
  MAX_TIMEOUT_SECONDS = 300
  # Polling timeout for checking git repo for other build statuses.
  SLEEP_TIMEOUT = 30

  # Sub-directories for LKGM and Chrome LKGM's.
  LKGM_SUBDIR = 'LKGM-candidates'
  CHROME_PFQ_SUBDIR = 'chrome-LKGM-candidates'

  def __init__(self, source_dir, checkout_repo, manifest_repo, branch,
               build_name, build_type, clobber=False,
               dry_run=True):
    """Initialize an LKGM Manager.

    Args:
      build_type:  Type of build.  Must be either chrome or binary.
    Other args see manifest_version.BuildSpecsManager.
    """
    super(LKGMManager, self).__init__(
        source_dir=source_dir, checkout_repo=checkout_repo,
        manifest_repo=manifest_repo, branch=branch, build_name=build_name,
        incr_type='branch', clobber=clobber, dry_run=dry_run)

    self.compare_versions_fn = _LKGMCandidateInfo.VersionCompare

    assert build_type in ('chrome', 'binary')
    if build_type == 'chrome':
      self.lkgm_subdir = self.CHROME_PFQ_SUBDIR
    else:
      self.lkgm_subdir = self.LKGM_SUBDIR

  def _LoadSpecs(self, version_info):
    """Loads the specifications from the working directory.
    Args:
      version_info: Info class for version information of cros.
    """
    super(LKGMManager, self)._LoadSpecs(version_info, self.lkgm_subdir)

  def _GetLatestCandidateByVersion(self, version_info):
    """Returns the latest lkgm candidate corresponding to the version file.
    Args:
      version_info: Info class for version information of cros.
    """
    if self.all:
      matched_lkgms = filter(
          lambda ver: ver.startswith(version_info.VersionString()), self.all)
      if matched_lkgms:
        return _LKGMCandidateInfo(sorted(matched_lkgms,
                                         key=self.compare_versions_fn)[-1])

    return _LKGMCandidateInfo(version_info.VersionString())


  def _SetInFlightWithRetry(self, commit_message, retries):
    for index in range(retries+1):
      try:
        self._SetInFlight(commit_message)
        break
      except (manifest_version.GitCommandException,
              cros_lib.RunCommandError) as e:
        last_error = 'Failed to set build in-flight: %s' % e
        logging.error(last_error)
        logging.error('Retrying:  Retry %d/%d' %
                      (index + 1, retries + 1))
    else:
      raise manifest_version.GenerateBuildSpecException(last_error)

  def CreateNewCandidate(self, version_file, retries=3):
    """Gets the version number of the next build spec to build.
      Args:
        version_file: File to use in cros when checking for cros version.
        retries: Number of retries for updating the status
      Returns:
        next_build: a string of the next build number for the builder to consume
                    or None in case of no need to build.
      Raises:
        GenerateBuildSpecException in case of failure to generate a buildspec
    """
    try:
      version_info = self._GetCurrentVersionInfo(version_file)
      self._LoadSpecs(version_info)
      lkgm_info = self._GetLatestCandidateByVersion(version_info)

      self.current_version = self._CreateNewBuildSpec(lkgm_info)
      if self.current_version:
        logging.debug('Using build spec: %s', self.current_version)
        commit_message = 'Automatic: Start %s %s' % (self.build_name,
                                                     self.current_version)
        self._SetInFlightWithRetry(commit_message, retries)

      return self.GetLocalManifest(self.current_version)

    except (cros_lib.RunCommandError,
            manifest_version.GitCommandException) as e:
      err_msg = 'Failed to generate LKGM Candidate. error: %s' % e
      logging.error(err_msg)
      raise manifest_version.GenerateBuildSpecException(err_msg)

  def GetLatestCandidate(self, version_file, retries=5):
    """Gets the version number of the next build spec to build.
      Args:
        version_file: File to use in cros when checking for cros version.
        retries: Number of retries for updating the status
      Returns:
{'fraction_non_alphanumeric': 0.08435268330622923, 'fraction_numerical': 0.003393905112069575, 'mean_word_length': 3.042297799371249, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7452556', 'n_tokens_mistral': 4361, 'n_tokens_neox': 4219, 'n_words': 1205}