<commit_before>from __future__ import unicode_literals

import json
import os
from pathlib import Path
from typing import Dict, List, Optional, Union

from common.methods import set_progress
from google.oauth2.credentials import Credentials
from googleapiclient.discovery import Resource as GCPResource
from googleapiclient.discovery import build
from googleapiclient.http import HttpError, MediaIoBaseUpload
from resourcehandlers.gcp.models import GCPHandler
from resources.models import Resource
from utilities.logger import ThreadLogger

logger = ThreadLogger(__name__)

FILE = "{{file}}"
MAKE_BLOB_PUBLIC = bool("{{make_blob_public}}")
api_dict = Dict[str, Union[str, List, Dict]]


# Helper functions for the main function below
def create_storage_api_wrapper(handler: GCPHandler) -> Optional[GCPResource]:
    """
    Using googleapiclient.discovery, build the api wrapper for the storage api.
    https://googleapis.github.io/google-api-python-client/docs/dyn/storage_v1.html
    """
    if not handler.gcp_api_credentials:
        set_progress(f"Handler {handler} is missing gcp api credentials.")
        return None

    credentials_dict = json.loads(handler.gcp_api_credentials)
    credentials = Credentials(**credentials_dict)

    set_progress(f"Connecting to GCP for handler: {handler}")
    storage_wrapper: GCPResource = build(
        "storage", "v1", credentials=credentials, cache_discovery=False
    )
    set_progress("Connection established")

    return storage_wrapper


def bucket_has_uniform_level_access(wrapper: GCPResource, bucket_name: str) -> bool:
    """
    If a bucket is set to control access to blobs, we can't specify the individual
    blob's public vs private setting.
    """
    bucket = wrapper.buckets().get(bucket=bucket_name).execute()
    logger.info(f"Got info for bucket: {bucket}")
    iam_configuration = bucket.get("iamConfiguration", {})
    access_configuration = iam_configuration.get("uniformBucketLevelAccess", {})
    has_uniform_level_access = access_configuration.get("enabled", False)
    return has_uniform_level_access


def upload_object(
    wrapper: GCPResource,
    bucket_name: str,
    object_name: str,
    file_location: str,
    make_public: bool,
):
    """
    Upload an object from a file to a bucket
    If a bucket isn't

    Media insertion:
    https://googleapis.github.io/google-api-python-client/docs/dyn/storage_v1.objects.html#insert
    Uploader:
    https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.http.MediaIoBaseUpload-class.html
    """
    upload_kwargs = {
        "mimetype": "application/octet-stream",
        "chunksize": 1024 * 1024,
        "resumable": False,
    }

    insert_kwargs = {
        "bucket": bucket_name,
        "body": {},
        "name": object_name,
    }

    if bucket_has_uniform_level_access(wrapper, bucket_name):
        set_progress(
            f"Bucket {bucket_name} has uniform bucket level access set. Ignoring "
            "privacy selection for the 'Make Blob Public' option. Read why at "
            "https://cloud.google.com/storage/docs/uniform-bucket-level-access"
        )
    else:
        insert_kwargs["predefinedAcl"] = "publicRead" if make_public else "private"

    set_progress(f"Opening file '{file_location}'")
    with open(file_location, "rb") as file:
        set_progress("Beginning to upload file.")
        media = MediaIoBaseUpload(file, **upload_kwargs)
        wrapper.objects().insert(**insert_kwargs, media_body=media).execute()

    set_progress("Upload complete!")


def _get_paginated_list_result(
    resource_method, collection_name: str, *args, **kwargs
) -> List[api_dict]:
    """
    Call a method on a collection on a resource that returns a paginated list.<commit_msg>[DEV-16198] Fix failure to upload as private<commit_after>from __future__ import unicode_literals

import json
import os
from pathlib import Path
from typing import Dict, List, Optional, Union

from common.methods import set_progress
from google.oauth2.credentials import Credentials
from googleapiclient.discovery import Resource as GCPResource
from googleapiclient.discovery import build
from googleapiclient.http import HttpError, MediaIoBaseUpload
from resourcehandlers.gcp.models import GCPHandler
from resources.models import Resource
from utilities.logger import ThreadLogger

logger = ThreadLogger(__name__)

FILE = "{{file}}"
MAKE_BLOB_PUBLIC = "{{make_blob_public}}" == "True"
api_dict = Dict[str, Union[str, List, Dict]]


# Helper functions for the main function below
def create_storage_api_wrapper(handler: GCPHandler) -> Optional[GCPResource]:
    """
    Using googleapiclient.discovery, build the api wrapper for the storage api.
    https://googleapis.github.io/google-api-python-client/docs/dyn/storage_v1.html
    """
    if not handler.gcp_api_credentials:
        set_progress(f"Handler {handler} is missing gcp api credentials.")
        return None

    credentials_dict = json.loads(handler.gcp_api_credentials)
    credentials = Credentials(**credentials_dict)

    set_progress(f"Connecting to GCP for handler: {handler}")
    storage_wrapper: GCPResource = build(
        "storage", "v1", credentials=credentials, cache_discovery=False
    )
    set_progress("Connection established")

    return storage_wrapper


def bucket_has_uniform_level_access(wrapper: GCPResource, bucket_name: str) -> bool:
    """
    If a bucket is set to control access to blobs, we can't specify the individual
    blob's public vs private setting.
    """
    bucket = wrapper.buckets().get(bucket=bucket_name).execute()
    logger.info(f"Got info for bucket: {bucket}")
    iam_configuration = bucket.get("iamConfiguration", {})
    access_configuration = iam_configuration.get("uniformBucketLevelAccess", {})
    has_uniform_level_access = access_configuration.get("enabled", False)
    return has_uniform_level_access


def upload_object(
    wrapper: GCPResource,
    bucket_name: str,
    object_name: str,
    file_location: str,
    make_public: bool,
):
    """
    Upload an object from a file to a bucket
    If a bucket isn't

    Media insertion:
    https://googleapis.github.io/google-api-python-client/docs/dyn/storage_v1.objects.html#insert
    Uploader:
    https://googleapis.github.io/google-api-python-client/docs/epy/googleapiclient.http.MediaIoBaseUpload-class.html
    """
    upload_kwargs = {
        "mimetype": "application/octet-stream",
        "chunksize": 1024 * 1024,
        "resumable": False,
    }

    insert_kwargs = {
        "bucket": bucket_name,
        "body": {},
        "name": object_name,
    }

    if bucket_has_uniform_level_access(wrapper, bucket_name):
        set_progress(
            f"Bucket {bucket_name} has uniform bucket level access set. Ignoring "
            "privacy selection for the 'Make Blob Public' option. Read why at "
            "https://cloud.google.com/storage/docs/uniform-bucket-level-access"
        )
    else:
        acl_value = "publicRead" if make_public else "private"
        insert_kwargs["predefinedAcl"] = acl_value
        set_progress(f"Setting access to file as {acl_value}")

    set_progress(f"Opening file '{file_location}'")
    with open(file_location, "rb") as file:
        set_progress("Beginning to upload file.")
        media = MediaIoBaseUpload(file, **upload_kwargs)
        wrapper.objects().insert(**insert_kwargs, media_body=media).execute()

    set_progress("Upload complete!")


def _get_paginated_list_result(
    resource_method, collection_name: str, *args, **kwargs
) -> List[api_dict]:
    """
    Call a method on a collection on a resource that returns a paginated list.
{'fraction_non_alphanumeric': 0.09809548656404905, 'fraction_numerical': 0.0037829376467518913, 'mean_word_length': 3.880330999363463, 'pattern_counts': {'":': 12, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 8, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2551836', 'n_tokens_mistral': 2284, 'n_tokens_neox': 2217, 'n_words': 572}