<commit_before>from snovault import (
    AfterModified,
    BeforeModified,
    CONNECTION,
    calculated_property,
    collection,
    load_schema,
    abstract_collection,
)
from snovault.schema_utils import schema_validator
from snovault.validators import validate_item_content_post
from snovault.attachment import ItemWithAttachment
from .base import (
    Item,
    collection_add,
)
from pyramid.httpexceptions import (
    HTTPForbidden,
    HTTPTemporaryRedirect,
    HTTPNotFound,
)
from pyramid.response import Response
from pyramid.settings import asbool
from pyramid.view import view_config
from urllib.parse import (
    parse_qs,
    urlparse,
)
import boto
from boto.exception import BotoServerError
import datetime
import json
import pytz
import os

import logging
logging.getLogger('boto').setLevel(logging.CRITICAL)
log = logging.getLogger(__name__)

BEANSTALK_ENV_PATH = "/opt/python/current/env"


def show_upload_credentials(request=None, context=None, status=None):
    if request is None or status not in ('uploading', 'to be uploaded by workflow', 'upload failed'):
        return False
    return request.has_permission('edit', context)


def force_beanstalk_env(profile_name, config_file=None):
    # set env variables if we are on elasticbeanstalk
    if not config_file:
        config_file = BEANSTALK_ENV_PATH
    if os.path.exists(config_file):
        if not os.environ.get("AWS_ACCESS_KEY_ID"):
            import subprocess
            command = ['bash', '-c', 'source ' + config_file + ' && env']
            proc = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
            for line in proc.stdout:
                key, _, value = line.partition("=")
                os.environ[key] = value[:-1]

            proc.communicate()

        conn = boto.connect_sts(aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
                                aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"))
        print("on beanstalk so adding environment variables")
    else:
        conn = boto.connect_sts(profile_name=profile_name)

    return conn


def external_creds(bucket, key, name=None, profile_name=None):
    '''
    if name is None, we want the link to s3 but no need to generate
    an access token.  This is useful for linking metadata to files that
    already exist on s3.
    '''

    import logging
    logging.getLogger('boto').setLevel(logging.CRITICAL)
    credentials = {}
    if name is not None:
        policy = {
            'Version': '2012-10-17',
            'Statement': [
                {
                    'Effect': 'Allow',
                    'Action': 's3:PutObject',
                    'Resource': 'arn:aws:s3:::{bucket}/{key}'.format(bucket=bucket, key=key),
                }
            ]
        }
<<<<<<< 97c139d7a11ebcacd7b4fd1ea5eec3a0d617576e
        boto.set_stream_logger('boto')
        conn = boto.connect_sts(profile_name=profile_name)
=======
        # boto.set_stream_logger('boto')
        conn = force_beanstalk_env(profile_name)
>>>>>>> auto pull credentials from beanstalk if they are present
        token = conn.get_federation_token(name, policy=json.dumps(policy))
        # 'access_key' 'secret_key' 'expiration' 'session_token'
        credentials = token.credentials.to_dict()
        credentials.update({
            'upload_url': 's3://{bucket}/{key}'.format(bucket=bucket, key=key),
            'federated_user_arn': token.federated_user_arn,
            'federated_user_id': token.federated_user_id,
            'request_id': token.request_id,
            'key': key
        })
    return {
        'service': 's3',
        'bucket': bucket,
        'key': key,
        'upload_credentials': credentials,
    }
<commit_msg>handle BotoServerError in upload_key calculated prop<commit_after>from snovault import (
    AfterModified,
    BeforeModified,
    CONNECTION,
    calculated_property,
    collection,
    load_schema,
    abstract_collection,
)
from snovault.schema_utils import schema_validator
from snovault.validators import validate_item_content_post
from snovault.attachment import ItemWithAttachment
from .base import (
    Item,
    collection_add,
)
from pyramid.httpexceptions import (
    HTTPForbidden,
    HTTPTemporaryRedirect,
    HTTPNotFound,
)
from pyramid.response import Response
from pyramid.settings import asbool
from pyramid.view import view_config
from urllib.parse import (
    parse_qs,
    urlparse,
)
import boto
from boto.exception import BotoServerError
import datetime
import json
import pytz
import os

import logging
logging.getLogger('boto').setLevel(logging.CRITICAL)
log = logging.getLogger(__name__)

BEANSTALK_ENV_PATH = "/opt/python/current/env"


def show_upload_credentials(request=None, context=None, status=None):
    if request is None or status not in ('uploading', 'to be uploaded by workflow', 'upload failed'):
        return False
    return request.has_permission('edit', context)


def force_beanstalk_env(profile_name, config_file=None):
    # set env variables if we are on elasticbeanstalk
    if not config_file:
        config_file = BEANSTALK_ENV_PATH
    if os.path.exists(config_file):
        if not os.environ.get("AWS_ACCESS_KEY_ID"):
            import subprocess
            command = ['bash', '-c', 'source ' + config_file + ' && env']
            proc = subprocess.Popen(command, stdout=subprocess.PIPE, universal_newlines=True)
            for line in proc.stdout:
                key, _, value = line.partition("=")
                os.environ[key] = value[:-1]

            proc.communicate()

        conn = boto.connect_sts(aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
                                aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"))
        print("on beanstalk so adding environment variables")
    else:
        conn = boto.connect_sts(profile_name=profile_name)

    return conn


def external_creds(bucket, key, name=None, profile_name=None):
    '''
    if name is None, we want the link to s3 but no need to generate
    an access token.  This is useful for linking metadata to files that
    already exist on s3.
    '''

    import logging
    logging.getLogger('boto').setLevel(logging.CRITICAL)
    credentials = {}
    if name is not None:
        policy = {
            'Version': '2012-10-17',
            'Statement': [
                {
                    'Effect': 'Allow',
                    'Action': 's3:PutObject',
                    'Resource': 'arn:aws:s3:::{bucket}/{key}'.format(bucket=bucket, key=key),
                }
            ]
        }
        # boto.set_stream_logger('boto')
        conn = force_beanstalk_env(profile_name)
        token = conn.get_federation_token(name, policy=json.dumps(policy))
        # 'access_key' 'secret_key' 'expiration' 'session_token'
        credentials = token.credentials.to_dict()
        credentials.update({
            'upload_url': 's3://{bucket}/{key}'.format(bucket=bucket, key=key),
            'federated_user_arn': token.federated_user_arn,
            'federated_user_id': token.federated_user_id,
            'request_id': token.request_id,
            'key': key
        })
    return {
        'service': 's3',
        'bucket': bucket,
        'key': key,
        'upload_credentials': credentials,
    }

{'fraction_non_alphanumeric': 0.10107468371650116, 'fraction_numerical': 0.006801795674057951, 'mean_word_length': 2.7916451779267666, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2068691', 'n_tokens_mistral': 2260, 'n_tokens_neox': 2182, 'n_words': 502}