<jupyter_start><jupyter_text>Query for dataThis notebook demonstrates the workflow to compile desired images by querying metadata using the database instance and downloading the images stored in blob storage.<jupyter_code># Cosmos DB config
config = {
    'ENDPOINT': os.environ.get('COSMOS_ENDPOINT'),
    'PRIMARYKEY': os.environ.get('COSMOS_KEY')
}

# Initialize the Cosmos client
client = cosmos_client.CosmosClient(url_connection=config['ENDPOINT'], auth={
                                    'masterKey': config['PRIMARYKEY']})

container_link = 'dbs/camera-trap/colls/images'  # database link + container link
with open('datasets.json') as f:
    datasets_table = json.load(f)
    
# this is a json object with the account name as key, and the key to the account as value
with open('blob_account_keys.json') as f:
    blob_account_keys = json.load(f)<jupyter_output><empty_output><jupyter_text>Select image entriesExample: top 1000 images from a given dataset with bounding boxes, selecting the file name and the dataset so we can plot the labels.<jupyter_code>%%time

dataset_name = 'rspb_gola'
# did not have species in many of these items
query = {'query': '''
SELECT TOP 1000 im.file_name, im.dataset, im.annotations.bbox, im.annotations.species
FROM images im
WHERE im.dataset = "{}" AND ARRAY_LENGTH(im.annotations.bbox) > 0
'''.format(dataset_name)}

options = {
    'enableCrossPartitionQuery': True
}

result_iterable = client.QueryItems(container_link, query, options, partition_key='idfg')
# if you want to restrict to one dataset, pass in partition_key=dataset

results = []
for item in iter(result_iterable):
    results.append(item)

print('Length of results:', len(results))
len(results)
results[77]<jupyter_output><empty_output><jupyter_text>Download images and visualize labelsFor large batches, download using `multiprocessing.ThreadPool`.<jupyter_code>sample_size = 2
sample_res = sample(results, sample_size)

for im in sample_res:
    dataset = im['dataset']
    storage_account = datasets_table[dataset]['storage_account']
    storage_container = datasets_table[dataset]['container']
    path_prefix = datasets_table[dataset]['path_prefix']

    print('Creating blob service')
    blob_service = BlockBlobService(account_name=storage_account, account_key=blob_account_keys[storage_account])
    print('Created')
    stream = io.BytesIO()
    _ = blob_service.get_blob_to_stream(storage_container, os.path.join(path_prefix, im['file_name']), stream)
    print('Downloaded')
    image = Image.open(stream)
    print('Opened')
    
    boxes = []
    classes = []
    
    for i in im['bbox']:
        boxes.append(i['bbox_rel'])
        classes.append(annotation_constants.bbox_category_name_to_id['animal'])

    visualization_utils.render_iMerit_boxes(boxes, classes, image)
    print('Visualized')
    image<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10638297872340426, 'fraction_numerical': 0.004185559818625741, 'mean_word_length': 5.076271186440678, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '64480', 'license': 'MIT', 'n_tokens_mistral': 887, 'n_tokens_neox': 863, 'n_words': 220, 'path': 'data_management/cosmos_db/query_for_data.ipynb', 'repo_name': 'CoreyJaskolski/CameraTraps', 'size': '2867'}