<commit_before>import atexit
import logging
import multiprocessing
import subprocess
import time
from urlparse import urlparse

import minio
from django.conf import settings
from minio import policy
from minio.error import BucketAlreadyOwnedByYou, ResponseError

logger = logging.getLogger(__name__)


def start_minio():
    """
    Start a minio subprocess, controlled by another thread.

    Returns the daemonized thread controlling the minio subprocess.
    """
    minio_process = multiprocessing.Process(target=_start_minio)
    minio_process.start()
    atexit.register(lambda: stop_minio(minio_process))
    return minio_process


def _start_minio():
    logger.info("Starting minio")

    MINIO_PROCESS = subprocess.Popen(
        ["run_minio.py"],
        stdin=subprocess.PIPE,
    )


def stop_minio(p):
    p.terminate()


def ensure_storage_bucket_public(bucket=None, will_sleep=True):

    # If true, sleep for 5 seconds to wait for minio to start
    if will_sleep:
        time.sleep(5)

    if not bucket:
        bucket = settings.AWS_STORAGE_BUCKET_NAME

    host = urlparse(settings.AWS_S3_ENDPOINT_URL).netloc

    c = minio.Minio(
        host,
        access_key=settings.AWS_ACCESS_KEY_ID,
        secret_key=settings.AWS_SECRET_ACCESS_KEY,
        secure=False
    )

    if not c.bucket_exists(bucket):
        try:
            c.make_bucket(bucket)
        except BucketAlreadyOwnedByYou:
            pass

    try:
        c.set_bucket_policy(bucket, "", policy.Policy.READ_ONLY)
        logger.debug("Successfully set the bucket policy to read only!")
    except ResponseError as e:
        logger.warning("Error setting bucket {} to readonly: {}".format(bucket, e))<commit_msg>use proper boto3 for setting the storage bucket policy<commit_after>import atexit
import logging
import multiprocessing
import subprocess
import time
from urlparse import urlparse

import boto3
import botocore
from django.conf import settings
from minio import policy
from minio.error import BucketAlreadyOwnedByYou, ResponseError

logger = logging.getLogger(__name__)


def start_minio():
    """
    Start a minio subprocess, controlled by another thread.

    Returns the daemonized thread controlling the minio subprocess.
    """
    minio_process = multiprocessing.Process(target=_start_minio)
    minio_process.start()
    atexit.register(lambda: stop_minio(minio_process))
    return minio_process


def _start_minio():
    logger.info("Starting minio")

    MINIO_PROCESS = subprocess.Popen(
        ["run_minio.py"],
        stdin=subprocess.PIPE,
    )


def stop_minio(p):
    p.terminate()


def ensure_storage_bucket_public(bucket=None, will_sleep=True):

    # If true, sleep for 5 seconds to wait for minio to start
    if will_sleep:
        time.sleep(5)

    if not bucket:
        bucketname = settings.AWS_STORAGE_BUCKET_NAME

    c = boto3.session.Session().client(
        service_name="s3",
        aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
        aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
        endpoint_url=settings.AWS_S3_ENDPOINT_URL,
    )

    try:
        c.head_bucket(Bucket=bucketname)
    except botocore.exceptions.ClientError as e:
        error_code = int(e.response["Error"]["Code"])
        if error_code == 404:
            c.create_bucket(Bucket=bucketname)

    c.put_bucket_acl(Bucket=bucketname, ACL="public-read")
    logger.debug("Successfully set the bucket policy to read only!")
{'fraction_non_alphanumeric': 0.07988422575976845, 'fraction_numerical': 0.0037626628075253256, 'mean_word_length': 3.5, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6955404', 'n_tokens_mistral': 1102, 'n_tokens_neox': 1069, 'n_words': 255}