<commit_before>import logging
import os
import gzip
import boto3

s3resource = boto3.resource('s3')
s3client = boto3.client('s3')
logger = logging.getLogger(__name__)

def delete_object(bucket, key):
    "Delete an object"
    s3resource.Object(bucket, key).delete()
    logger.debug('deleted object: {0}'.format(key))

def get_object(bucket, key):
    "Download an object into memory"
    data = s3resource.Object(bucket, key).get()["Body"].read()
    logger.debug('downlated object: {0}'.format(key))
    return data

def copy_object(src_bucket, src_key, dst_bucket, dst_key):
    "Copy an object to a new S3 location"
    src_path = os.path.join(src_bucket, src_key)
    s3resource.Object(dst_bucket, dst_key).copy_from(CopySource=src_path)
    logger.debug('copying object from: {0} to:{1}'.format(src_key, dst_key))

def exists_object(bucket, key):
    "Check if an object exists in S3"
    results = s3client.list_objects(Bucket=bucket, Prefix=key)
    return 'Contents' in results

def download_object(bucket, key, file_name=None):
    "Download object to local file"
    if file_name is None:
        file_name = key.split('/')[-1]
    s3resource.Bucket(bucket).download_file(Key=key, Filename=file_name)
    logger.debug('downloaded object: {0}'.format(key))

def get_gzip_object(bucket, key):
    "Download and un-gzip object into memory"
    datagzip = get_object(bucket, key)
    data_b = gzip.decompress(data_gzip)
    logger.debug('downloaded gziped object: {0}'.format(key))
    return data_b.decode("utf-8")

def download_gzip_object(bucket, key):
    "Download and un-gzip object to local file"
    file_path = key.split('/')[-1]
    download_object(bucket, key, file_path)
    out_file = un_gzip(file_path)
    logger.debug('downloaded gziped object: {0}'.format(key))
    return out_file

def un_gzip(file_path):
    "Un-gzip local file"
    if file_path.endswith('.gz'):
        out_file = file_path.replace('.gz', '')
    else:
        out_file = file_path + '.un-gziped'
    with gzip.open(file_path, 'rb') as gzip_f, open(out_file, 'wb') as out_f:
        # refactor with itorator
        data_b = gzip_f.read()
        out_f.write(data_b)
    return out_file

def upload_file(bucket, key, local_file):
    "Upload & gzip local file to S3"
    gzip_file = local_file + '.gz'
    with open(local_file, 'rb') as file_in, gzip.open(gzip_file, 'wb') as gzip_out:
        gzip_out.writelines(file_in)
    try:
        s3resource.Bucket(bucket).upload_file(
            Filename = gzip_file, 
            Key = key, 
            ExtraArgs = {<commit_msg>minor<commit_after>import logging
import os
import gzip
import boto3

s3resource = boto3.resource('s3')
s3client = boto3.client('s3')
logger = logging.getLogger(__name__)

def delete_object(bucket, key):
    "Delete an object"
    s3resource.Object(bucket, key).delete()
    logger.debug('deleted object: {0}'.format(key))

def get_object(bucket, key):
    "Download an object into memory"
    data = s3resource.Object(bucket, key).get()["Body"].read()
    logger.debug('downlated object: {0}'.format(key))
    return data

def copy_object(src_bucket, src_key, dst_bucket, dst_key):
    "Copy an object to a new S3 location"
    src_path = os.path.join(src_bucket, src_key)
    s3resource.Object(dst_bucket, dst_key).copy_from(CopySource=src_path)
    logger.debug('copying object from: {0} to:{1}'.format(src_key, dst_key))

def exists_object(bucket, key):
    "Check if an object exists in S3"
    results = s3client.list_objects(Bucket=bucket, Prefix=key)
    return 'Contents' in results

def download_object(bucket, key, file_name=None):
    "Download object to local file"
    if file_name is None:
        file_name = key.split('/')[-1]
    s3resource.Bucket(bucket).download_file(Key=key, Filename=file_name)
    logger.debug('downloaded object: {0}'.format(key))

def get_gzip_object(bucket, key):
    "Download and un-gzip object into memory"
    datagzip = get_object(bucket, key)
    data_b = gzip.decompress(data_gzip)
    logger.debug('downloaded gziped object: {0}'.format(key))
    return data_b.decode("utf-8")

def download_gzip_object(bucket, key):
    "Download and un-gzip object to local file"
    file_path = key.split('/')[-1]
    download_object(bucket, key, file_path)
    out_file = un_gzip(file_path)
    logger.debug('downloaded gziped object: {0}'.format(key))
    return out_file

def un_gzip(file_path):
    "Un-gzip local file"
    if file_path.endswith('.gz'):
        out_file = file_path.replace('.gz', '')
    else:
        out_file = file_path + '.un-gziped'
    with gzip.open(file_path, 'rb') as gzip_f, open(out_file, 'wb') as out_f:
        data_b = gzip_f.read()
        out_f.write(data_b)
    return out_file

def upload_file(bucket, key, local_file):
    "Upload & gzip local file to S3"
    gzip_file = local_file + '.gz'
    with open(local_file, 'rb') as file_in, gzip.open(gzip_file, 'wb') as gzip_out:
        gzip_out.writelines(file_in)
    try:
        s3resource.Bucket(bucket).upload_file(
            Filename = gzip_file, 
            Key = key, 
            ExtraArgs = {
{'fraction_non_alphanumeric': 0.1272299549108018, 'fraction_numerical': 0.010194079592236817, 'mean_word_length': 3.7284522706209455, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 22, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4842740', 'n_tokens_mistral': 1793, 'n_tokens_neox': 1783, 'n_words': 407}