<commit_before># -*- coding: utf-8 -*-
'''
Manage S3 Buckets
=================

.. versionadded:: 2016.3.0

Create and destroy S3 buckets. Be aware that this interacts with Amazon's services,
and so may incur charges.

This module uses ``boto3``, which can be installed via package, or pip.

This module accepts explicit vpc credentials but can also utilize
IAM roles assigned to the instance through Instance Profiles. Dynamic
credentials are then automatically obtained from AWS API and no further
configuration is necessary. More information available `here
<http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html>`_.

If IAM roles are not used you need to specify them either in a pillar file or
in the minion's config file:

.. code-block:: yaml

    vpc.keyid: <KEY>
    vpc.key: <KEY>

It's also possible to specify ``key``, ``keyid`` and ``region`` via a profile,
either passed in as a dict, or as a string to pull from pillars or minion
config:

.. code-block:: yaml

    myprofile:
        keyid: <KEY>
        key: <KEY>
            region: us-east-1

.. code-block:: yaml

    Ensure bucket exists:
        boto_s3_bucket.present:
            - Bucket: mybucket
            - LocationConstraint: EU
            - ACL:
              - GrantRead: "uri=http://acs.amazonaws.com/groups/global/AllUsers"
            - CORSRules:
              - AllowedHeaders: []
                AllowedMethods: ["GET"]
                AllowedOrigins: ["*"]
                ExposeHeaders: []
                MaxAgeSeconds: 123
            - LifecycleConfiguration:
              - Expiration:
                  Days: 123
                ID: "idstring"
                Prefix: "prefixstring"
                Status: "enabled",
                ID: "lc1"
                Transitions:
                  - Days: 123
                    StorageClass: "GLACIER"
                NoncurrentVersionTransitions:
                  - NoncurrentDays: 123
                    StorageClass: "GLACIER"
                NoncurrentVersionExpiration:
                  NoncurrentDays: 123
            - Logging:
                TargetBucket: log_bucket
                TargetPrefix: prefix
                TargetGrants:
                  - Grantee:
                      DisplayName: "string"
                      EmailAddress: "string"
                      ID: "string"
                      Type: "AmazonCustomerByEmail"
                      URI: "string"
                    Permission: "READ"
            - NotificationConfiguration:
                LambdaFunctionConfiguration:
                  - Id: "string"
                    LambdaFunctionArn: "string"
                    Events:
                      - "s3:ObjectCreated:*"
                    Filter:
                      Key:
                        FilterRules:
                          - Name: "prefix"
                            Value: "string"
            - Policy:
                Version: "2012-10-17"
                Statement:
                  - Sid: "String"
                    Effect: "Allow"
                    Principal:
                      AWS: "arn:aws:iam::133434421342:root"
                    Action: "s3:PutObject"
                    Resource: "arn:aws:s3:::my-bucket/*"
            - Replication:
                Role: myrole
                Rules:
                  - ID: "string"
                    Prefix: "string"
                    Status: "Enabled"
                    Destination:
                      Bucket: "arn:aws:s3:::my-bucket"
            - RequestPayment:
                Payer: Requester
            - Tagging:
                tag_name: tag_value
                tag_name_2: tag_value
            - Versioning:
                Status: "Enabled"
            - Website:
                ErrorDocument:
                  Key: "error.html"
                IndexDocument:
                  Suffix: "index.html"
                RedirectAllRequestsTo:
                  Hostname: "string"
                  Protocol: "http"
                RoutingRules:
                  - Condition:
                      HttpErrorCodeReturnedEquals: "string"
                      KeyPrefixEquals: "string"
                    Redirect:
                      HostName: "string"
                      HttpRedirectCode: "string"
                      Protocol: "http"
                      ReplaceKeyPrefixWith: "string"
                      ReplaceKeyWith: "string"
            - region: us-east-1
            - keyid: <KEY>
            - key: <KEY>

'''

# Import Python Libs
from __future__ import absolute_import
import logging
import os
import os.path
from copy import deepcopy
import json

# Import Salt Libs
from salt.ext.six import string_types  # pylint: disable=import-error
from salt.utils.boto3 import json_objs_equal

log = logging.getLogger(__name__)


def __virtual__():
    '''
    Only load if boto is available.
    '''
    return 'boto_s3_bucket' if 'boto_s3_bucket.exists' in __salt__ else False


def _normalize_user(user_dict):
    ret = deepcopy(user_dict)
    # 'Type' is required as input to the AWS API, but not returned as output. So
    # we ignore it everywhere.
    if 'Type' in ret:
        del ret['Type']
    return ret


def _get_canonical_id(region, key, keyid, profile):
    ret = __salt__['boto_s3_bucket.list'](
        region=region, key=key, keyid=keyid, profile=profile
    ).get('Owner')
    return _normalize_user(ret)


def _prep_acl_for_compare(ACL):
    '''
    Prepares the ACL returned from the AWS API for comparison with a given one.
    '''
    ret = deepcopy(ACL)
    ret['Owner'] = _normalize_user(ret['Owner'])
    for item in ACL.get('Grants', ()):
        item['Grantee'] = _normalize_user(item.get('Grantee'))
    return ret


def _acl_to_grant(ACL, owner_canonical_id):<commit_msg>Fixed bug where _prep_acl_for_compare() would edit but not return<commit_after># -*- coding: utf-8 -*-
'''
Manage S3 Buckets
=================

.. versionadded:: 2016.3.0

Create and destroy S3 buckets. Be aware that this interacts with Amazon's services,
and so may incur charges.

This module uses ``boto3``, which can be installed via package, or pip.

This module accepts explicit vpc credentials but can also utilize
IAM roles assigned to the instance through Instance Profiles. Dynamic
credentials are then automatically obtained from AWS API and no further
configuration is necessary. More information available `here
<http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html>`_.

If IAM roles are not used you need to specify them either in a pillar file or
in the minion's config file:

.. code-block:: yaml

    vpc.keyid: <KEY>
    vpc.key: <KEY>

It's also possible to specify ``key``, ``keyid`` and ``region`` via a profile,
either passed in as a dict, or as a string to pull from pillars or minion
config:

.. code-block:: yaml

    myprofile:
        keyid: <KEY>
        key: <KEY>
            region: us-east-1

.. code-block:: yaml

    Ensure bucket exists:
        boto_s3_bucket.present:
            - Bucket: mybucket
            - LocationConstraint: EU
            - ACL:
              - GrantRead: "uri=http://acs.amazonaws.com/groups/global/AllUsers"
            - CORSRules:
              - AllowedHeaders: []
                AllowedMethods: ["GET"]
                AllowedOrigins: ["*"]
                ExposeHeaders: []
                MaxAgeSeconds: 123
            - LifecycleConfiguration:
              - Expiration:
                  Days: 123
                ID: "idstring"
                Prefix: "prefixstring"
                Status: "enabled",
                ID: "lc1"
                Transitions:
                  - Days: 123
                    StorageClass: "GLACIER"
                NoncurrentVersionTransitions:
                  - NoncurrentDays: 123
                    StorageClass: "GLACIER"
                NoncurrentVersionExpiration:
                  NoncurrentDays: 123
            - Logging:
                TargetBucket: log_bucket
                TargetPrefix: prefix
                TargetGrants:
                  - Grantee:
                      DisplayName: "string"
                      EmailAddress: "string"
                      ID: "string"
                      Type: "AmazonCustomerByEmail"
                      URI: "string"
                    Permission: "READ"
            - NotificationConfiguration:
                LambdaFunctionConfiguration:
                  - Id: "string"
                    LambdaFunctionArn: "string"
                    Events:
                      - "s3:ObjectCreated:*"
                    Filter:
                      Key:
                        FilterRules:
                          - Name: "prefix"
                            Value: "string"
            - Policy:
                Version: "2012-10-17"
                Statement:
                  - Sid: "String"
                    Effect: "Allow"
                    Principal:
                      AWS: "arn:aws:iam::133434421342:root"
                    Action: "s3:PutObject"
                    Resource: "arn:aws:s3:::my-bucket/*"
            - Replication:
                Role: myrole
                Rules:
                  - ID: "string"
                    Prefix: "string"
                    Status: "Enabled"
                    Destination:
                      Bucket: "arn:aws:s3:::my-bucket"
            - RequestPayment:
                Payer: Requester
            - Tagging:
                tag_name: tag_value
                tag_name_2: tag_value
            - Versioning:
                Status: "Enabled"
            - Website:
                ErrorDocument:
                  Key: "error.html"
                IndexDocument:
                  Suffix: "index.html"
                RedirectAllRequestsTo:
                  Hostname: "string"
                  Protocol: "http"
                RoutingRules:
                  - Condition:
                      HttpErrorCodeReturnedEquals: "string"
                      KeyPrefixEquals: "string"
                    Redirect:
                      HostName: "string"
                      HttpRedirectCode: "string"
                      Protocol: "http"
                      ReplaceKeyPrefixWith: "string"
                      ReplaceKeyWith: "string"
            - region: us-east-1
            - keyid: <KEY>
            - key: <KEY>

'''

# Import Python Libs
from __future__ import absolute_import
import logging
import os
import os.path
from copy import deepcopy
import json

# Import Salt Libs
from salt.ext.six import string_types  # pylint: disable=import-error
from salt.utils.boto3 import json_objs_equal

log = logging.getLogger(__name__)


def __virtual__():
    '''
    Only load if boto is available.
    '''
    return 'boto_s3_bucket' if 'boto_s3_bucket.exists' in __salt__ else False


def _normalize_user(user_dict):
    ret = deepcopy(user_dict)
    # 'Type' is required as input to the AWS API, but not returned as output. So
    # we ignore it everywhere.
    if 'Type' in ret:
        del ret['Type']
    return ret


def _get_canonical_id(region, key, keyid, profile):
    ret = __salt__['boto_s3_bucket.list'](
        region=region, key=key, keyid=keyid, profile=profile
    ).get('Owner')
    return _normalize_user(ret)


def _prep_acl_for_compare(ACL):
    '''
    Prepares the ACL returned from the AWS API for comparison with a given one.
    '''
    ret = deepcopy(ACL)
    ret['Owner'] = _normalize_user(ret['Owner'])
    for item in ret.get('Grants', ()):
        item['Grantee'] = _normalize_user(item.get('Grantee'))
    return ret


def _acl_to_grant(ACL, owner_canonical_id):
{'fraction_non_alphanumeric': 0.0874466268146883, 'fraction_numerical': 0.010247651579846286, 'mean_word_length': 1.4659928405980207, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4866311', 'n_tokens_mistral': 3289, 'n_tokens_neox': 2982, 'n_words': 853}