<jupyter_start><jupyter_text>Setup<jupyter_code>!pip install ibm-cos-sdk==2.6 python-dotenv
import os

from dotenv import load_dotenv

load_dotenv()

bucket = os.getenv("BUCKET", "")
access_key_id = os.getenv("ACCESS_KEY_ID", "")
secret_access_key = os.getenv("SECRET_ACCESS_KEY", "")
endpoint_url = os.getenv("ENDPOINT_URL", "")
import os
import shutil

if os.path.exists('tmp') and os.path.isdir('tmp'):
    shutil.rmtree('tmp')

if os.path.exists('model_web') and os.path.isdir('model_web'):
    shutil.rmtree('model_web')

CLOUD_ANNOTATIONS_DATA = os.path.join('tmp', bucket)

os.makedirs(CLOUD_ANNOTATIONS_DATA, exist_ok=True)

import json
import ibm_boto3

def download_file_cos(local_file_name, key): 
    '''
    Wrapper function to download a file from cloud object storage using the
    credential dict provided and loading it into memory
    '''
    cos = ibm_boto3.client(
        service_name='s3',
        aws_access_key_id=access_key_id,
        aws_secret_access_key=secret_access_key,
        endpoint_url=endpoint_url)
    try:
        res=cos.download_file(Bucket=bucket, Key=key, Filename=local_file_name)
    except Exception as e:
        print('Exception', e)
    else:
        print('File Downloaded')

def get_annotations(): 
    cos = ibm_boto3.client(
        service_name='s3',
        aws_access_key_id=access_key_id,
        aws_secret_access_key=secret_access_key,
        endpoint_url=endpoint_url)
    try:
        return json.loads(cos.get_object(Bucket=bucket, Key='_annotations.json')['Body'].read())
    except Exception as e:
        print('Exception', e)
annotations = get_annotations()

download_file_cos(os.path.join(CLOUD_ANNOTATIONS_DATA, '_annotations.json'), '_annotations.json')

for image in annotations['annotations'].keys():
    local_path = os.path.join(CLOUD_ANNOTATIONS_DATA, image)
    download_file_cos(local_path, image)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11758530183727034, 'fraction_numerical': 0.003674540682414698, 'mean_word_length': 4.540697674418604, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '60001', 'license': 'MIT', 'n_tokens_mistral': 660, 'n_tokens_neox': 655, 'n_words': 116, 'path': 'download-data.ipynb', 'repo_name': 'cloud-annotations/elyra-object-detection-training', 'size': '1905'}