<commit_before>#   Foremast - Pipeline Tooling
#
#   Copyright 2016 Gogo, LLC
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Deploy artifacts to S3."""
import logging
import os
import subprocess

from ..exceptions import S3ArtifactNotFound
from ..utils import get_details, get_properties

LOG = logging.getLogger(__name__)


class S3Deployment(object):
    """Handle uploading artifacts to S3 and S3 deployment strategies."""

    def __init__(self, app, env, region, prop_path, artifact_path, artifact_version):
        """S3 deployment object.

        Args:
            app (str): Application name
            env (str): Environment/Account
            region (str): AWS Region
            prop_path (str): Path of environment property file
            artifact_path (str): Path to tar.gz artifact
        """
        self.app_name = app
        self.env = env
        self.region = region
        self.artifact_path = artifact_path
        self.version = artifact_version
        self.properties = get_properties(prop_path)
        self.s3props = self.properties[self.env]['s3']
        generated = get_details(app=app, env=env)

        if self.s3props.get('shared_bucket_master'):
            self.bucket = generated.shared_s3_app_bucket()
            self.s3path = app
        elif self.s3props.get('shared_bucket_target'):
            shared_app = self.s3props['shared_bucket_target']
            newgenerated = get_details(app=shared_app, env=env)
            self.bucket = newgenerated.shared_s3_app_bucket()
            self.s3path = app
        else:
            self.bucket = generated.s3_app_bucket()
            self.s3path = self.s3props['path'].lstrip('/')

        self.s3_version_uri = ''
        self.s3_latest_uri = ''
        self.setup_pathing()

    def setup_pathing(self):
        """Format pathing for S3 deployments."""
        self.s3_version_uri = self._path_formatter(self.version)
        self.s3_latest_uri = self._path_formatter("LATEST")
        self.s3_canary_uri = self._path_formatter("CANARY")
        self.s3_alpha_uri = self._path_formatter("ALPHA")

    def _path_formatter(self, suffix):
        """Format the s3 path properly.

        Args:
            suffix (str): suffix to add on to an s3 path

        Returns:
            str: formatted path

        """
        path_format = "{}/{}/{}"
        s3_format = "s3://{}"
        path = path_format.format(self.bucket, self.s3path, suffix)
        formatted_path = path.replace('//', '/')  # removes configuration errors
        full_path = s3_format.format(formatted_path)
        return full_path

    def upload_artifacts(self):
        """Upload artifacts to S3 and copy to LATEST depending on strategy."""
        deploy_strategy = self.properties[self.env]["deploy_strategy"]
        self._upload_artifacts_to_version()
        if deploy_strategy == "highlander":
            self._sync_to_uri(self.s3_latest_uri)
        elif deploy_strategy == "canary":
            self._sync_to_uri(self.s3_canary_uri)
        elif deploy_strategy == "alpha":
            self._sync_to_uri(self.s3_alpha_uri)
        else:
            raise NotImplementedError

    def promote_artifacts(self, promote_stage='latest'):
        """Promote artifact version to dest.

        Args:
            promote_stage (string): Stage that is being promoted
        """
        if promote_stage.lower() == 'alpha':
            self._sync_to_uri(self.s3_canary_uri)
        else:
            self._sync_to_uri(self.s3_latest_uri)

    def _upload_artifacts_to_version(self):
        """Recursively upload directory contents to S3."""
        if not os.listdir(self.artifact_path) or not self.artifact_path:
            raise S3ArtifactNotFound

        uploaded = False
        if self.s3props.get("content_metadata"):
            LOG.info("Uploading in multiple parts to set metadata")
            uploaded = self.content_metadata_uploads()

        if not uploaded:
            cmd = 'aws s3 sync {} {} --delete --exact-timestamps --profile {}'.format(self.artifact_path,<commit_msg>explicitly promote canary to latest<commit_after>#   Foremast - Pipeline Tooling
#
#   Copyright 2016 Gogo, LLC
#
#   Licensed under the Apache License, Version 2.0 (the "License");
#   you may not use this file except in compliance with the License.
#   You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""Deploy artifacts to S3."""
import logging
import os
import subprocess

from ..exceptions import S3ArtifactNotFound
from ..utils import get_details, get_properties

LOG = logging.getLogger(__name__)


class S3Deployment(object):
    """Handle uploading artifacts to S3 and S3 deployment strategies."""

    def __init__(self, app, env, region, prop_path, artifact_path, artifact_version):
        """S3 deployment object.

        Args:
            app (str): Application name
            env (str): Environment/Account
            region (str): AWS Region
            prop_path (str): Path of environment property file
            artifact_path (str): Path to tar.gz artifact
        """
        self.app_name = app
        self.env = env
        self.region = region
        self.artifact_path = artifact_path
        self.version = artifact_version
        self.properties = get_properties(prop_path)
        self.s3props = self.properties[self.env]['s3']
        generated = get_details(app=app, env=env)

        if self.s3props.get('shared_bucket_master'):
            self.bucket = generated.shared_s3_app_bucket()
            self.s3path = app
        elif self.s3props.get('shared_bucket_target'):
            shared_app = self.s3props['shared_bucket_target']
            newgenerated = get_details(app=shared_app, env=env)
            self.bucket = newgenerated.shared_s3_app_bucket()
            self.s3path = app
        else:
            self.bucket = generated.s3_app_bucket()
            self.s3path = self.s3props['path'].lstrip('/')

        self.s3_version_uri = ''
        self.s3_latest_uri = ''
        self.setup_pathing()

    def setup_pathing(self):
        """Format pathing for S3 deployments."""
        self.s3_version_uri = self._path_formatter(self.version)
        self.s3_latest_uri = self._path_formatter("LATEST")
        self.s3_canary_uri = self._path_formatter("CANARY")
        self.s3_alpha_uri = self._path_formatter("ALPHA")

    def _path_formatter(self, suffix):
        """Format the s3 path properly.

        Args:
            suffix (str): suffix to add on to an s3 path

        Returns:
            str: formatted path

        """
        path_format = "{}/{}/{}"
        s3_format = "s3://{}"
        path = path_format.format(self.bucket, self.s3path, suffix)
        formatted_path = path.replace('//', '/')  # removes configuration errors
        full_path = s3_format.format(formatted_path)
        return full_path

    def upload_artifacts(self):
        """Upload artifacts to S3 and copy to LATEST depending on strategy."""
        deploy_strategy = self.properties[self.env]["deploy_strategy"]
        self._upload_artifacts_to_version()
        if deploy_strategy == "highlander":
            self._sync_to_uri(self.s3_latest_uri)
        elif deploy_strategy == "canary":
            self._sync_to_uri(self.s3_canary_uri)
        elif deploy_strategy == "alpha":
            self._sync_to_uri(self.s3_alpha_uri)
        else:
            raise NotImplementedError

    def promote_artifacts(self, promote_stage='latest'):
        """Promote artifact version to dest.

        Args:
            promote_stage (string): Stage that is being promoted
        """
        if promote_stage.lower() == 'alpha':
            self._sync_to_uri(self.s3_canary_uri)
        elif promote_stage.lower() == 'canary':
            self._sync_to_uri(self.s3_latest_uri)
        else:
            self._sync_to_uri(self.s3_latest_uri)

    def _upload_artifacts_to_version(self):
        """Recursively upload directory contents to S3."""
        if not os.listdir(self.artifact_path) or not self.artifact_path:
            raise S3ArtifactNotFound

        uploaded = False
        if self.s3props.get("content_metadata"):
            LOG.info("Uploading in multiple parts to set metadata")
            uploaded = self.content_metadata_uploads()

        if not uploaded:
            cmd = 'aws s3 sync {} {} --delete --exact-timestamps --profile {}'.format(self.artifact_path,
{'fraction_non_alphanumeric': 0.09633620689655173, 'fraction_numerical': 0.010668103448275863, 'mean_word_length': 2.5545767905017236, 'pattern_counts': {'":': 6, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2945773', 'n_tokens_mistral': 2721, 'n_tokens_neox': 2632, 'n_words': 693}