<commit_before># Copyright 2020 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilities for deploying jobs to GCP."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import subprocess
import uuid

from googleapiclient import discovery
from googleapiclient import errors

from . import gcp
from . import machine_config

try:
    from tensorflow.python.framework.versions import VERSION
except ImportError:
    # Use TF runtime version 2.1 (latest supported) as the default.
    # https://cloud.google.com/ai-platform/training/docs/runtime-version-list#tpu-support
    VERSION = "2.1"


def deploy_job(
    region,
    image_uri,
    chief_config,
    worker_count,
    worker_config,
    entry_point_args,
    enable_stream_logs,
    job_labels={},
):
    """Deploys job with the given parameters to Google Cloud.

    Args:
        region: GCP region name.
        image_uri: The docker image uri.
        chief_config: `MachineConfig` that represents the configuration for
            the chief worker in a distribution cluster.
        worker_count: Integer that represents the number of general workers
            in a distribution cluster. This count does not include the
            chief worker.
        worker_config: `MachineConfig` that represents the configuration for
            the general workers in a distribution cluster.
        entry_point_args: Command line arguments to pass to the
            `entry_point` program.
        enable_stream_logs: Boolean flag which when enabled streams logs
            back from the cloud job.

    Raises:
        RuntimeError, if there was an error submitting the job.
    """
    job_id = _generate_job_id()
    project_id = gcp.get_project_name()
    ml_apis = discovery.build("ml", "v1", cache_discovery=False)

    request_dict = _create_request_dict(
        job_id,
        region,
        image_uri,
        chief_config,
        worker_count,
        worker_config,
        entry_point_args,
        job_labels=job_labels,
    )
    try:
        response = (
            ml_apis.projects()
            .jobs()
            .create(parent="projects/{}".format(project_id), body=request_dict)
            .execute()
        )
        _print_logs_info(job_id, project_id)
        if enable_stream_logs:
            _stream_logs(job_id)
    except errors.HttpError as err:
        raise RuntimeError("There was an error submitting the job." + err._get_reason())
    return job_id


def _create_request_dict(
    job_id,
    region,
    image_uri,
    chief_config,
    worker_count,
    worker_config,
    entry_point_args,
    job_labels={},
):
    """Creates request dictionary for the CAIP training service.

    Args:
        job_id: String, unique job id.
        region: GCP region name.
        image_uri: The docker image uri.
        chief_config: `MachineConfig` that represents the configuration for
            the chief worker in a distribution cluster.
        worker_count: Integer that represents the number of general workers
            in a distribution cluster. This count does not include the
            chief worker.
        worker_config: `MachineConfig` that represents the configuration for
            the general workers in a distribution cluster.
        entry_point_args: Command line arguments to pass to the
            `entry_point` program.
        job_labels: Dict of str: str. Labels to organize jobs. See 
            https://cloud.google.com/ai-platform/training/docs/resource-labels.

    Returns:
        The job request dictionary.
    """
    training_input = {}
    training_input["region"] = region
    training_input["scaleTier"] = "custom"
    training_input["masterType"] = gcp.get_machine_type(
        chief_config.cpu_cores, chief_config.memory, chief_config.accelerator_type
    )

    # Set master config
    chief_machine_config = {}
    chief_machine_config["imageUri"] = image_uri
    chief_machine_config["acceleratorConfig"] = {}
    chief_machine_config["acceleratorConfig"]["count"] = str(
        chief_config.accelerator_count
    )
    chief_machine_config["acceleratorConfig"]["type"] = gcp.get_accelerator_type(
        chief_config.accelerator_type.value
    )

    training_input["masterConfig"] = chief_machine_config
    training_input["workerCount"] = str(worker_count)

    if worker_count > 0:
        training_input["workerType"] = gcp.get_machine_type(
            worker_config.cpu_cores,
            worker_config.memory,
            worker_config.accelerator_type,
        )

        worker_machine_config = {}
        worker_machine_config["imageUri"] = image_uri
        worker_machine_config["acceleratorConfig"] = {}
        worker_machine_config["acceleratorConfig"]["count"] = str(
            worker_config.accelerator_count
        )
        worker_machine_config["acceleratorConfig"]["type"] = gcp.get_accelerator_type(
            worker_config.accelerator_type.value
        )

        if machine_config.is_tpu_config(worker_config):
            # TPU is not support for TF version > 2.1.0
            if VERSION >= "2.2":
                raise NotImplementedError(
                    "TPUs are only supported for TF version <= 2.1.0"
                )

            # AI Platform runtime version spec is required for training<commit_msg>remove changes on deploy.py<commit_after># Copyright 2020 Google LLC. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Utilities for deploying jobs to GCP."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import subprocess
import uuid

from googleapiclient import discovery
from googleapiclient import errors

from . import gcp
from . import machine_config

try:
    from tensorflow.python.framework.versions import VERSION
except ImportError:
    # Use TF runtime version 2.1 (latest supported) as the default.
    # https://cloud.google.com/ai-platform/training/docs/runtime-version-list#tpu-support
    VERSION = "2.1"


def deploy_job(
    region,
    image_uri,
    chief_config,
    worker_count,
    worker_config,
    entry_point_args,
    enable_stream_logs,
    job_labels={},
):
    """Deploys job with the given parameters to Google Cloud.

    Args:
        region: GCP region name.
        image_uri: The docker image uri.
        chief_config: `MachineConfig` that represents the configuration for
            the chief worker in a distribution cluster.
        worker_count: Integer that represents the number of general workers
            in a distribution cluster. This count does not include the
            chief worker.
        worker_config: `MachineConfig` that represents the configuration for
            the general workers in a distribution cluster.
        entry_point_args: Command line arguments to pass to the
            `entry_point` program.
        enable_stream_logs: Boolean flag which when enabled streams logs
            back from the cloud job.

    Raises:
        RuntimeError, if there was an error submitting the job.
    """
    job_id = _generate_job_id()
    project_id = gcp.get_project_name()
    ml_apis = discovery.build("ml", "v1", cache_discovery=False)

    request_dict = _create_request_dict(
        job_id,
        region,
        image_uri,
        chief_config,
        worker_count,
        worker_config,
        entry_point_args,
        job_labels=job_labels,
    )
    try:
        response = (
            ml_apis.projects()
            .jobs()
            .create(parent="projects/{}".format(project_id), body=request_dict)
            .execute()
        )
        _print_logs_info(job_id, project_id)
        if enable_stream_logs:
            _stream_logs(job_id)
    except errors.HttpError as err:
        raise RuntimeError("There was an error submitting the job." + err._get_reason())
    return job_id


def _create_request_dict(
    job_id,
    region,
    image_uri,
    chief_config,
    worker_count,
    worker_config,
    entry_point_args,
    job_labels={},
):
    """Creates request dictionary for the CAIP training service.

    Args:
        job_id: String, unique job id.
        region: GCP region name.
        image_uri: The docker image uri.
        chief_config: `MachineConfig` that represents the configuration for
            the chief worker in a distribution cluster.
        worker_count: Integer that represents the number of general workers
            in a distribution cluster. This count does not include the
            chief worker.
        worker_config: `MachineConfig` that represents the configuration for
            the general workers in a distribution cluster.
        entry_point_args: Command line arguments to pass to the
            `entry_point` program.
        job_labels: Dict of str: str. Labels to organize jobs. See 
            https://cloud.google.com/ai-platform/training/docs/resource-labels.

    Returns:
        The job request dictionary.
    """
    training_input = {}
    training_input["region"] = region
    training_input["scaleTier"] = "custom"
    training_input["masterType"] = gcp.get_machine_type(
        chief_config.cpu_cores, chief_config.memory, chief_config.accelerator_type
    )

    # Set master config
    chief_machine_config = {}
    chief_machine_config["imageUri"] = image_uri
    chief_machine_config["acceleratorConfig"] = {}
    chief_machine_config["acceleratorConfig"]["count"] = str(
        chief_config.accelerator_count
    )
    chief_machine_config["acceleratorConfig"]["type"] = gcp.get_accelerator_type(
        chief_config.accelerator_type.value
    )

    training_input["masterConfig"] = chief_machine_config
    training_input["workerCount"] = str(worker_count)

    if worker_count > 0:
        training_input["workerType"] = gcp.get_machine_type(
            worker_config.cpu_cores,
            worker_config.memory,
            worker_config.accelerator_type,
        )

        worker_machine_config = {}
        worker_machine_config["imageUri"] = image_uri
        worker_machine_config["acceleratorConfig"] = {}
        worker_machine_config["acceleratorConfig"]["count"] = str(
            worker_config.accelerator_count
        )
        worker_machine_config["acceleratorConfig"]["type"] = gcp.get_accelerator_type(
            worker_config.accelerator_type.value
        )

        if machine_config.is_tpu_config(worker_config):
            # AI Platform runtime version spec is required for training
{'fraction_non_alphanumeric': 0.07190447347927663, 'fraction_numerical': 0.003114995240979493, 'mean_word_length': 2.788266142248443, 'pattern_counts': {'":': 1, '<': 4, '<?xml version=': 0, '>': 7, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1777539', 'n_tokens_mistral': 3165, 'n_tokens_neox': 3098, 'n_words': 947}