<commit_before># Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Implements Spin CLI support commands for buildtool."""

import copy
import logging
import os
import re

from collections import namedtuple
from distutils.version import LooseVersion

from buildtool import (
    DEFAULT_BUILD_NUMBER,
    SPIN_REPOSITORY_NAMES,

    BomSourceCodeManager,
    BranchSourceCodeManager,
    CommandProcessor,
    CommandFactory,
    GitRunner,
    RepositoryCommandFactory,
    RepositoryCommandProcessor,
    check_subprocess,
    check_subprocesses_to_logfile,
    check_options_set,
    raise_and_log_error,
    ConfigError)

from google.cloud import storage


class DistArch(namedtuple('DistArch', ['dist', 'arch'])):
  """Describes an distribution/architecture pair.
  """
  def __repl__(self):
    return '{},{}'.format(self.dist, self.arch)

  @property
  def filename(self):
    return 'spin.exe' if self.dist == 'windows' else 'spin'


DIST_ARCH_LIST = [
    DistArch('darwin', 'amd64'),
    DistArch('linux', 'amd64'),
    DistArch('windows', 'amd64')
]


class BuildSpinCommand(RepositoryCommandProcessor):
  def __init__(self, factory, options, **kwargs):
    super(BuildSpinCommand, self).__init__(
      factory, options, source_repository_names=SPIN_REPOSITORY_NAMES, **kwargs)
    options_copy = copy.copy(options)
    self.__gcs_uploader = SpinGcsUploader(options)
    self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir())
    self.__build_version = None  # recorded after build
    bom_contents = BomSourceCodeManager.load_bom(options)
    gate_entry = bom_contents.get('services', {}).get('gate', {})
    if not gate_entry:
      raise_and_log_error(
          ConfigError('No gate service entry found in bom {}'.format(bom_contents)))
    self.__gate_version = gate_entry['version']

  def _do_can_skip_repository(self, repository):
    self.source_code_manager.ensure_local_repository(repository)
    source_info = self.source_code_manager.refresh_source_info(
      repository, self.options.build_number)
    self.__build_version = source_info.to_build_version()

    version_exists = [
      self.__gcs_uploader.check_file_exists(
        'spin/{}/{}/{}/{}'.format(self.__build_version, d.dist, d.arch, d.filename))
      for d in DIST_ARCH_LIST
    ]
    all_exist = all(version_exists)
    if all_exist:
      logging.info('Skipping spin CLI build since all versions exist for build version %s', self.__build_version)
    return all_exist

  def build_all_distributions(self, repository):
    spin_package_path = repository.origin
    double_slash = spin_package_path.find('//')
    # Trim spin_package_path to format go package path properly.
    if spin_package_path.find('//') != -1:
      spin_package_path = spin_package_path[double_slash+2:]
    if spin_package_path[:-1] == '/':
      spin_package_path = spin_package_path[:-1]

    source_info = self.source_code_manager.refresh_source_info(
      repository, self.options.build_number)
    self.__build_version = source_info.to_build_version()

    # NOTE: go build is opinionated about where the source repository lives --
    # it expects the source repository to be under $GOPATH/src/<package path>,
    # and will use that the source code in that directory *even if you clone
    # your own copy of the source repo*.
    #
    # To facilitate go's pattern and maintain buildtool's
    # build_input -> build_output pattern, we override go's GOPATH to our
    # filesystem destinations.

    gopath = os.path.abspath(self.get_input_dir())
    config_root = os.path.join(gopath, 'src', spin_package_path)

    env = dict(os.environ)
    env.update({'GOPATH': gopath})

    # spin source + dependency update.
    for dist_arch in DIST_ARCH_LIST:
      env.update({'GOOS': dist_arch.dist,
                  'GOARCH': dist_arch.arch})
      check_subprocess('go get -v -u {}'.format(spin_package_path), cwd=gopath, env=env)

    for dist_arch in DIST_ARCH_LIST:
      # GCS sub-directory the binaries are stored in are specified by<commit_msg>chore(buildtool): Log "go get" when building spin<commit_after># Copyright 2018 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Implements Spin CLI support commands for buildtool."""

import copy
import logging
import os
import re

from collections import namedtuple
from distutils.version import LooseVersion

from buildtool import (
    DEFAULT_BUILD_NUMBER,
    SPIN_REPOSITORY_NAMES,

    BomSourceCodeManager,
    BranchSourceCodeManager,
    CommandProcessor,
    CommandFactory,
    GitRunner,
    RepositoryCommandFactory,
    RepositoryCommandProcessor,
    check_subprocess,
    check_subprocesses_to_logfile,
    check_options_set,
    raise_and_log_error,
    ConfigError)

from google.cloud import storage


class DistArch(namedtuple('DistArch', ['dist', 'arch'])):
  """Describes an distribution/architecture pair.
  """
  def __repl__(self):
    return '{},{}'.format(self.dist, self.arch)

  @property
  def filename(self):
    return 'spin.exe' if self.dist == 'windows' else 'spin'


DIST_ARCH_LIST = [
    DistArch('darwin', 'amd64'),
    DistArch('linux', 'amd64'),
    DistArch('windows', 'amd64')
]


class BuildSpinCommand(RepositoryCommandProcessor):
  def __init__(self, factory, options, **kwargs):
    super(BuildSpinCommand, self).__init__(
      factory, options, source_repository_names=SPIN_REPOSITORY_NAMES, **kwargs)
    options_copy = copy.copy(options)
    self.__gcs_uploader = SpinGcsUploader(options)
    self.__scm = BranchSourceCodeManager(options_copy, self.get_input_dir())
    self.__build_version = None  # recorded after build
    bom_contents = BomSourceCodeManager.load_bom(options)
    gate_entry = bom_contents.get('services', {}).get('gate', {})
    if not gate_entry:
      raise_and_log_error(
          ConfigError('No gate service entry found in bom {}'.format(bom_contents)))
    self.__gate_version = gate_entry['version']

  def _do_can_skip_repository(self, repository):
    self.source_code_manager.ensure_local_repository(repository)
    source_info = self.source_code_manager.refresh_source_info(
      repository, self.options.build_number)
    self.__build_version = source_info.to_build_version()

    version_exists = [
      self.__gcs_uploader.check_file_exists(
        'spin/{}/{}/{}/{}'.format(self.__build_version, d.dist, d.arch, d.filename))
      for d in DIST_ARCH_LIST
    ]
    all_exist = all(version_exists)
    if all_exist:
      logging.info('Skipping spin CLI build since all versions exist for build version %s', self.__build_version)
    return all_exist

  def build_all_distributions(self, repository):
    spin_package_path = repository.origin
    double_slash = spin_package_path.find('//')
    # Trim spin_package_path to format go package path properly.
    if spin_package_path.find('//') != -1:
      spin_package_path = spin_package_path[double_slash+2:]
    if spin_package_path[:-1] == '/':
      spin_package_path = spin_package_path[:-1]

    source_info = self.source_code_manager.refresh_source_info(
      repository, self.options.build_number)
    self.__build_version = source_info.to_build_version()

    # NOTE: go build is opinionated about where the source repository lives --
    # it expects the source repository to be under $GOPATH/src/<package path>,
    # and will use that the source code in that directory *even if you clone
    # your own copy of the source repo*.
    #
    # To facilitate go's pattern and maintain buildtool's
    # build_input -> build_output pattern, we override go's GOPATH to our
    # filesystem destinations.

    gopath = os.path.abspath(self.get_input_dir())
    config_root = os.path.join(gopath, 'src', spin_package_path)

    env = dict(os.environ)
    env.update({'GOPATH': gopath})

    # spin source + dependency update.
    for dist_arch in DIST_ARCH_LIST:
      env.update({'GOOS': dist_arch.dist,
                  'GOARCH': dist_arch.arch})
      context = '%s-%s' % (dist_arch.dist, dist_arch.arch)
      logfile = self.get_logfile_path(
          repository.name + '-go-get-' + context)
      labels = {'repository': repository.name,
                'dist': dist_arch.dist,
                'arch': dist_arch.arch}
      cmd = 'go get -v -u {}'.format(spin_package_path)
      self.metrics.time_call(
          'GoGet', labels, self.metrics.default_determine_outcome_labels,
          check_subprocesses_to_logfile, 'Fetching Go packages ' + context,
          logfile, [cmd], cwd=gopath, env=env)

    for dist_arch in DIST_ARCH_LIST:
      # GCS sub-directory the binaries are stored in are specified by
{'fraction_non_alphanumeric': 0.09790354229061242, 'fraction_numerical': 0.00371785603635237, 'mean_word_length': 3.8884401817264007, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5500476', 'n_tokens_mistral': 2947, 'n_tokens_neox': 2862, 'n_words': 785}