<commit_before>#!/usr/bin/python
#
# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import datetime
import os
import sys
import yaml

from annotate_source import Annotator
from spinnaker.run import run_quick

DEPENDENCIES = 'dependencies'
SERVICES = 'services'
VERSION = 'version'

# Dependency versions.
CONSUL_VERSION = '0.7.5'
REDIS_VERSION = '2:2.8.4-2'
VAULT_VERSION = '0.7.0'

GOOGLE_CONTAINER_BUILDER_SERVICE_BASE_CONFIG = {
  'steps': [
    {
      'name': 'java:8',
      'env': ['GRADLE_USER_HOME=cache'],
      'args': []
    },
    {
      'name': 'gcr.io/cloud-builders/docker',
      'args': []
    }
  ],
  'images': [],
  'timeout': '3600s'
}

GOOGLE_CONTAINER_BUILDER_MONITORING_BASE_CONFIG = {
  'steps': [
    {
      'name': 'gcr.io/cloud-builders/docker',
      'dir': 'spinnaker-monitoring-daemon',
      'args': []
    }
  ],
  'images': [],
  'timeout': '3600s'
}

class BomGenerator(Annotator):
  """Provides facilities for generating the Bill of Materials file for the
  Spinnaker product release.

  This assumes Halyard (https://github.com/spinnaker/halyard) is installed on
  the machine this script runs on.
  """

  COMPONENTS = [
    'clouddriver',
    'deck',
    'echo',
    'front50',
    'gate',
    'igor',
    'orca',
    'rosco',
    'fiat',
    'spinnaker-monitoring',
    'spinnaker'
  ]

  def __init__(self, options):
    self.__base_dir = options.base_dir
    self.__docker_registry = options.docker_registry
    self.__bom_file = ''
    self.__component_versions = {}
    self.__changelog_start_hashes = {} # Hashes to start from when generating changelogs.
    self.__toplevel_version = ''
    self.__changelog_output = options.changelog_output
    self.__alias = options.bom_alias
    super(BomGenerator, self).__init__(options)

  @property
  def base_dir(self):
    return self.__base_dir

  @classmethod
  def init_argument_parser(cls, parser):
    """Initialize command-line arguments."""
    parser.add_argument('--base_dir', default='.', required=True,
                        help="Base directory containing the component's git repositories as subdirectories.")
    parser.add_argument('--container_builder', default='gcb',
                        help="Type of builder to use. Currently, the supported options are {'gcb', 'docker'}.")
    parser.add_argument('--docker_registry', default='',
                        help="Docker registry to push the container images to.")
    parser.add_argument('--changelog_output', default='',
                        help="Output file to write the changelog to.")
    parser.add_argument('--bom_alias', default='',
                        help="Alias to rename the 'real' BOM as. This also sets the Spinnaker version as the alias.")
    super(BomGenerator, cls).init_argument_parser(parser)

  def __version_from_tag(self, comp):
    """Determine the component version from the 'version-X.Y.Z' git tag.

    Args:
      comp [string]: Spinnaker component name.

    Returns:
      [string] Component version with build number and without 'version-'.
    """
    version_bump = self.__component_versions[comp]
    next_tag_with_build = '{0}-{1}'.format(version_bump.version_str,
                                           self.build_number)
    first_dash_idx = next_tag_with_build.index('-')
    return next_tag_with_build[first_dash_idx + 1:]

  def write_container_builder_gcr_config(self):
    """Write a configuration file for producing Container Images with Google Container Builder for each microservice.
    """
    for comp in self.__component_versions:
      if comp == 'spinnaker-monitoring':
        config = dict(GOOGLE_CONTAINER_BUILDER_MONITORING_BASE_CONFIG)
        version = self.__version_from_tag(comp)
        versioned_image = '{reg}/monitoring-daemon:{tag}'.format(reg=self.__docker_registry,
                                                                 tag=version)
        config['steps'][0]['args'] = ['build', '-t', versioned_image, '-f', 'Dockerfile', '.']
        config['images'] = [versioned_image]
        config_file = '{0}-gcb.yml'.format(comp)
        with open(config_file, 'w') as cfg:
          yaml.dump(config, cfg, default_flow_style=True)
      elif comp == 'spinnaker':
        pass
      else:
        config = dict(GOOGLE_CONTAINER_BUILDER_SERVICE_BASE_CONFIG)
        gradle_version = self.__version_from_tag(comp)
        gradle_cmd = ''
        if comp == 'deck':
          gradle_cmd = './gradlew build -PskipTests'
        else:
          gradle_cmd = './gradlew {0}-web:installDist -x test'.format(comp)
        config['steps'][0]['args'] = ['bash', '-c', gradle_cmd]
        versioned_image = '{reg}/{repo}:{tag}'.format(reg=self.__docker_registry,
                                                      repo=comp,
                                                      tag=gradle_version)
        config['steps'][1]['args'] = ['build', '-t', versioned_image, '-f', 'Dockerfile.slim', '.']
        config['images'] = [versioned_image]
        config_file = '{0}-gcb.yml'.format(comp)
        with open(config_file, 'w') as cfg:
            yaml.dump(config, cfg, default_flow_style=True)

  def write_docker_version_files(self):
    """Write a file containing the full tag for each microservice for Docker.
    """
    for comp in self.__component_versions:
      if comp == 'spinnaker':
        pass
      gradle_version = self.__version_from_tag(comp)
      docker_tag = '{reg}/{comp}:{tag}'.format(reg=self.__docker_registry,
                                               comp=comp,
                                               tag=gradle_version)

      config_file = '{0}-docker.yml'.format(comp)
      with open(config_file, 'w') as cfg:
        cfg.write(docker_tag)

  def generate_changelog(self):
    """Generate a release changelog and write it to a file.

    The changelog contains a section per microservice that describes the<commit_msg>fix(prevalidate): Support for building patch releases. (#1626)<commit_after>#!/usr/bin/python
#
# Copyright 2017 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
import datetime
import os
import sys
import yaml

from annotate_source import Annotator
from spinnaker.run import run_quick

DEPENDENCIES = 'dependencies'
SERVICES = 'services'
VERSION = 'version'

# Dependency versions.
CONSUL_VERSION = '0.7.5'
REDIS_VERSION = '2:2.8.4-2'
VAULT_VERSION = '0.7.0'

GOOGLE_CONTAINER_BUILDER_SERVICE_BASE_CONFIG = {
  'steps': [
    {
      'name': 'java:8',
      'env': ['GRADLE_USER_HOME=cache'],
      'args': []
    },
    {
      'name': 'gcr.io/cloud-builders/docker',
      'args': []
    }
  ],
  'images': [],
  'timeout': '3600s'
}

GOOGLE_CONTAINER_BUILDER_MONITORING_BASE_CONFIG = {
  'steps': [
    {
      'name': 'gcr.io/cloud-builders/docker',
      'dir': 'spinnaker-monitoring-daemon',
      'args': []
    }
  ],
  'images': [],
  'timeout': '3600s'
}

class BomGenerator(Annotator):
  """Provides facilities for generating the Bill of Materials file for the
  Spinnaker product release.

  This assumes Halyard (https://github.com/spinnaker/halyard) is installed on
  the machine this script runs on.
  """

  COMPONENTS = [
    'clouddriver',
    'deck',
    'echo',
    'front50',
    'gate',
    'igor',
    'orca',
    'rosco',
    'fiat',
    'spinnaker-monitoring',
    'spinnaker'
  ]

  def __init__(self, options):
    self.__base_dir = options.base_dir
    self.__docker_registry = options.docker_registry
    self.__bom_file = ''
    self.__component_versions = {}
    self.__changelog_start_hashes = {} # Hashes to start from when generating changelogs.
    self.__toplevel_version = ''
    self.__changelog_output = options.changelog_output
    self.__alias = options.bom_alias
    super(BomGenerator, self).__init__(options)

  @property
  def base_dir(self):
    return self.__base_dir

  @classmethod
  def init_argument_parser(cls, parser):
    """Initialize command-line arguments."""
    parser.add_argument('--base_dir', default='.', required=True,
                        help="Base directory containing the component's git repositories as subdirectories.")
    parser.add_argument('--container_builder', default='gcb',
                        help="Type of builder to use. Currently, the supported options are {'gcb', 'docker'}.")
    parser.add_argument('--docker_registry', default='',
                        help="Docker registry to push the container images to.")
    parser.add_argument('--changelog_output', default='',
                        help="Output file to write the changelog to.")
    parser.add_argument('--bom_alias', default='',
                        help="Alias to rename the 'real' BOM as. This also sets the Spinnaker version as the alias.")
    super(BomGenerator, cls).init_argument_parser(parser)

  def __version_from_tag(self, comp):
    """Determine the component version from the 'version-X.Y.Z' git tag.

    Args:
      comp [string]: Spinnaker component name.

    Returns:
      [string] Component version with build number and without 'version-'.
    """
    version_bump = self.__component_versions[comp]
    next_tag_with_build = '{0}-{1}'.format(version_bump.version_str,
                                           self.build_number)
    first_dash_idx = next_tag_with_build.index('-')
    return next_tag_with_build[first_dash_idx + 1:]

  def write_container_builder_gcr_config(self):
    """Write a configuration file for producing Container Images with Google Container Builder for each microservice.
    """
    for comp in self.__component_versions:
      if comp == 'spinnaker-monitoring':
        config = dict(GOOGLE_CONTAINER_BUILDER_MONITORING_BASE_CONFIG)
        version = self.__version_from_tag(comp)
        versioned_image = '{reg}/monitoring-daemon:{tag}'.format(reg=self.__docker_registry,
                                                                 tag=version)
        config['steps'][0]['args'] = ['build', '-t', versioned_image, '-f', 'Dockerfile', '.']
        config['images'] = [versioned_image]
        config_file = '{0}-gcb.yml'.format(comp)
        with open(config_file, 'w') as cfg:
          yaml.dump(config, cfg, default_flow_style=True)
      elif comp == 'spinnaker':
        pass
      else:
        config = dict(GOOGLE_CONTAINER_BUILDER_SERVICE_BASE_CONFIG)
        gradle_version = self.__version_from_tag(comp)

        # Gradle complains if the git tag version doesn't match the branch's version, i.e.
        # in patch releases. As a workaround, we checkout the HEAD commit of the
        # current branch in a 'detached HEAD' state so Gradle doesn't complain about
        # our branch version and git tag version not matching.
        gradle_cmd = 'git rev-parse HEAD | xargs git checkout ;'
        if comp == 'deck':
          gradle_cmd += ' ./gradlew build -PskipTests'
        else:
          gradle_cmd += ' ./gradlew {0}-web:installDist -x test'.format(comp)
        config['steps'][0]['args'] = ['bash', '-c', gradle_cmd]
        versioned_image = '{reg}/{repo}:{tag}'.format(reg=self.__docker_registry,
                                                      repo=comp,
                                                      tag=gradle_version)
        config['steps'][1]['args'] = ['build', '-t', versioned_image, '-f', 'Dockerfile.slim', '.']
        config['images'] = [versioned_image]
        config_file = '{0}-gcb.yml'.format(comp)
        with open(config_file, 'w') as cfg:
            yaml.dump(config, cfg, default_flow_style=True)

  def write_docker_version_files(self):
    """Write a file containing the full tag for each microservice for Docker.
    """
    for comp in self.__component_versions:
      if comp == 'spinnaker':
        pass
      gradle_version = self.__version_from_tag(comp)
      docker_tag = '{reg}/{comp}:{tag}'.format(reg=self.__docker_registry,
                                               comp=comp,
                                               tag=gradle_version)

      config_file = '{0}-docker.yml'.format(comp)
      with open(config_file, 'w') as cfg:
        cfg.write(docker_tag)

  def generate_changelog(self):
    """Generate a release changelog and write it to a file.

    The changelog contains a section per microservice that describes the
{'fraction_non_alphanumeric': 0.11682278006744098, 'fraction_numerical': 0.006294492319220682, 'mean_word_length': 2.7341913822048127, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1158914', 'n_tokens_mistral': 4043, 'n_tokens_neox': 3901, 'n_words': 1012}