<commit_before>#!/usr/bin/python

# Copyright 2017 Google Inc. All rights reserved.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
from datetime import datetime
import json
import logging
from ruamel import yaml
import subprocess
import sys

from google.cloud import storage


def main():
    logging.getLogger().setLevel(logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument('--infile', '-i',
                        help='templated cloudbuild config file.',
                        required=True)
    parser.add_argument('--bucket', '-b',
                        help='GCS bucket to publish runtime to',
                        default='runtime-builders')
    parser.add_argument('--builder-name',
                        help='the name of the runtime or project '
                        'associated with this builder',
                        required=True)
    args = parser.parse_args()

    templated_file_contents = _resolve_tags(args.infile)
    logging.info(templated_file_contents)
    logging.info(_publish_to_gcs(templated_file_contents,
                                 args.builder_name,
                                 args.bucket))


def _resolve_tags(config_file):
    """
    Given a templated YAML cloudbuild config file, parse it, resolve image tags
    on each build step's image to the corresponding digest, and write new
    config with fully qualified images to temporary file for upload to GCS.

    Keyword arguments:
    config_file -- string representing path to
    templated cloudbuild YAML config file

    Return value:
    path to temporary file containing fully qualified config file, to be
    published to GCS.
    """
    with open(config_file, 'r') as infile:
        try:
            config = yaml.round_trip_load(infile)

            for step in config.get('steps'):<commit_msg>consolidate config files into single directory (#98)<commit_after>#!/usr/bin/python

# Copyright 2017 Google Inc. All rights reserved.

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at

#     http://www.apache.org/licenses/LICENSE-2.0

# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse
from datetime import datetime
import glob
import json
import logging
import os
from ruamel import yaml
import subprocess
import sys

from google.cloud import storage


def main():
    logging.getLogger().setLevel(logging.INFO)
    parser = argparse.ArgumentParser()
    parser.add_argument('--directory', '-d',
                        help='templated cloudbuild config file.',
                        required=True)
    parser.add_argument('--bucket', '-b',
                        help='GCS bucket to publish runtime to',
                        default='runtime-builders')
    args = parser.parse_args()

    return _resolve_and_publish(args.directory, args.bucket)


def _resolve_and_publish(directory, bucket):
    try:
        gcs_paths = []
        for filepath in glob.glob(os.path.join(directory, '*.json')):
            with open(filepath, 'r') as f:
                project_cfg = json.load(f)
                project_name = project_cfg['project']
                for builder in project_cfg['builders']:
                    cfg = os.path.abspath(str(builder['path']))
                    name = builder['name']
                    builder_name = project_name + '_' + name

                    templated_file = _resolve_tags(cfg)
                    logging.info(templated_file)
                    gcs_paths.append(_publish_to_gcs(templated_file,
                                                     builder_name,
                                                     bucket))

        logging.info('Published Runtimes:')
        logging.info(gcs_paths)
    except ValueError as ve:
        logging.error('Error when parsing JSON! Check file formatting. \n{0}'
                      .format(ve))
    except KeyError as ke:
        logging.error('Config file is missing required field! \n{0}'
                      .format(ke))


def _resolve_tags(config_file):
    """
    Given a templated YAML cloudbuild config file, parse it, resolve image tags
    on each build step's image to the corresponding digest, and write new
    config with fully qualified images to temporary file for upload to GCS.

    Keyword arguments:
    config_file -- string representing path to
    templated cloudbuild YAML config file

    Return value:
    path to temporary file containing fully qualified config file, to be
    published to GCS.
    """
    with open(config_file, 'r') as infile:
        try:
            config = yaml.round_trip_load(infile)

            for step in config.get('steps'):
{'fraction_non_alphanumeric': 0.07702020202020202, 'fraction_numerical': 0.0036075036075036075, 'mean_word_length': 2.370820668693009, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '654176', 'n_tokens_mistral': 1430, 'n_tokens_neox': 1366, 'n_words': 513}