<commit_before>from fabric.decorators import task, roles

from haus_vars import with_vars, APP_INFO, parse_vars
from fabric.api import run, execute
from fabric.context_managers import cd

from heroku import create_fixture_on_s3, grab_fixture_on_s3

import cStringIO



def copyBucket(srcBucketName, dstBucketName, aws_key, aws_secret, folder_name='uploads'):
    from boto.s3.connection import S3Connection
    conn = S3Connection(aws_key, aws_secret)
    source = conn.get_bucket(srcBucketName);
    destination = conn.get_bucket(dstBucketName);
    if folder_name:
        s3keys = source.list(folder_name)
    else:
        s3keys = source.list()

    for k in s3keys:
        print 'Copying ' + k.key + ' from ' + srcBucketName + ' to ' + dstBucketName
        destination.copy_key(k.key, srcBucketName, k.key, preserve_acl=True)


def copyBucketDifferentOwners(src_settings, dst_settings, folder_name='uploads'):
    from boto.s3.connection import S3Connection
    srcBucketName = src_settings['AWS_BUCKET_NAME']
    dstBucketName = dst_settings['AWS_BUCKET_NAME']
    src_conn = S3Connection(src_settings['AWS_ACCESS_KEY_ID'], src_settings['AWS_SECRET_ACCESS_KEY'])
    dst_conn = S3Connection(dst_settings['AWS_ACCESS_KEY_ID'], dst_settings['AWS_SECRET_ACCESS_KEY'])
    source = src_conn.get_bucket(srcBucketName);
    destination = dst_conn.get_bucket(dstBucketName);
    if folder_name:
        s3keys = source.list(folder_name)
    else:
        s3keys = source.list()

    for k in s3keys:
        print 'Copying ' + k.key + ' from ' + srcBucketName + ' to ' + dstBucketName
        f = cStringIO.StringIO()
        k.get_contents_to_file(f)
        nk = destination.new_key(k.key)
        f.seek(0)
        nk.set_contents_from_file(f, 
                                  policy='public-read', 
                                  headers={ 'Content-Type': k.content_type }
                                  )


@task
@roles('vagrant')
def update_uploads(src_env='staging', dst_env='dev', different_owners=False):
    """copies the uploads folder from src_env to dst_env s3 buckets, pass different_owners=True to copy between s3 buckets that belong to different accounts"""
    different_owners = str(diffrent_owners).lowercase() == 'true'
    print "{}".format(APP_INFO)
    src_app_env = APP_INFO[src_env]['APP_ENV']
    dst_app_env = APP_INFO[dst_env]['APP_ENV']
    with(cd('/var/www')):
        src_settings = parse_vars(run("APP_ENV={} ./manage.py settings_vars AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_BUCKET_NAME".format(src_app_env)))
        dst_settings = parse_vars(run("APP_ENV={} ./manage.py settings_vars AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_BUCKET_NAME".format(dst_app_env)))
    src = src_settings['AWS_BUCKET_NAME']
    dst = dst_settings['AWS_BUCKET_NAME']
    folder = 'uploads'


    owners = {
              'dev': 1,
              'staging': 1,<commit_msg>Update s3_copy.py<commit_after>from fabric.decorators import task, roles

from haus_vars import with_vars, APP_INFO, parse_vars
from fabric.api import run, execute
from fabric.context_managers import cd

from heroku import create_fixture_on_s3, grab_fixture_on_s3

import cStringIO



def copyBucket(srcBucketName, dstBucketName, aws_key, aws_secret, folder_name='uploads'):
    from boto.s3.connection import S3Connection
    conn = S3Connection(aws_key, aws_secret)
    source = conn.get_bucket(srcBucketName);
    destination = conn.get_bucket(dstBucketName);
    if folder_name:
        s3keys = source.list(folder_name)
    else:
        s3keys = source.list()

    for k in s3keys:
        print 'Copying ' + k.key + ' from ' + srcBucketName + ' to ' + dstBucketName
        destination.copy_key(k.key, srcBucketName, k.key, preserve_acl=True)


def copyBucketDifferentOwners(src_settings, dst_settings, folder_name='uploads'):
    from boto.s3.connection import S3Connection
    srcBucketName = src_settings['AWS_BUCKET_NAME']
    dstBucketName = dst_settings['AWS_BUCKET_NAME']
    src_conn = S3Connection(src_settings['AWS_ACCESS_KEY_ID'], src_settings['AWS_SECRET_ACCESS_KEY'])
    dst_conn = S3Connection(dst_settings['AWS_ACCESS_KEY_ID'], dst_settings['AWS_SECRET_ACCESS_KEY'])
    source = src_conn.get_bucket(srcBucketName);
    destination = dst_conn.get_bucket(dstBucketName);
    if folder_name:
        s3keys = source.list(folder_name)
    else:
        s3keys = source.list()

    for k in s3keys:
        print 'Copying ' + k.key + ' from ' + srcBucketName + ' to ' + dstBucketName
        f = cStringIO.StringIO()
        k.get_contents_to_file(f)
        nk = destination.new_key(k.key)
        f.seek(0)
        nk.set_contents_from_file(f, 
                                  policy='public-read', 
                                  headers={ 'Content-Type': k.content_type }
                                  )


@task
@roles('vagrant')
def update_uploads(src_env='staging', dst_env='dev', different_owners=False):
    """copies the uploads folder from src_env to dst_env s3 buckets, pass different_owners=True to copy between s3 buckets that belong to different accounts"""
    different_owners = str(different_owners).lowercase() == 'true'
    print "{}".format(APP_INFO)
    src_app_env = APP_INFO[src_env]['APP_ENV']
    dst_app_env = APP_INFO[dst_env]['APP_ENV']
    with(cd('/var/www')):
        src_settings = parse_vars(run("APP_ENV={} ./manage.py settings_vars AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_BUCKET_NAME".format(src_app_env)))
        dst_settings = parse_vars(run("APP_ENV={} ./manage.py settings_vars AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_BUCKET_NAME".format(dst_app_env)))
    src = src_settings['AWS_BUCKET_NAME']
    dst = dst_settings['AWS_BUCKET_NAME']
    folder = 'uploads'


    owners = {
              'dev': 1,
              'staging': 1,
{'fraction_non_alphanumeric': 0.10322247113561951, 'fraction_numerical': 0.007065311046010684, 'mean_word_length': 3.5664830841856805, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '872513', 'n_tokens_mistral': 1945, 'n_tokens_neox': 1903, 'n_words': 360}