<jupyter_start><jupyter_text>Pulls image data from covid-chestxray-datasetPulls image data from https://github.com/ieee8023/covid-chestxray-dataset.git and outputs a zip folder containting the images and a csv file containing the image metadata<jupyter_code>!pip3 install wget==3.2 gitpython==3.1.14 ansible==3.1.0
import wget
wget.download(
    'https://raw.githubusercontent.com/' +
    'elyra-ai/component-library/master/claimed_utils.py'
)
from claimed_utils import zipdir
import os
import git
from ansible.module_utils.parsing.convert_bool import boolean
from io import BytesIO
from minio import Minio
import shutil
# @param output_filename
# @param metadata_filename
# @returns images.zip
# @returns metadata.csv
output_filename = os.environ.get('output_filename', 'images.zip')
metadata_filename = os.environ.get('metadata_filename', 'metadata.csv')
checkpoint = boolean(os.environ.get('checkpoint', False))
checkpoint_ip = os.environ.get('checkpoint_ip')
checkpoint_user = os.environ.get('checkpoint_user', 'minio')
checkpoint_pass = os.environ.get('checkpoint_pass', '<PASSWORD>')
checkpoint_bucket = os.environ.get('checkpoint_bucket', 'checkpoint')
exists = False

if checkpoint:
    client = Minio(checkpoint_ip,
                   checkpoint_user,
                   checkpoint_pass,
                   secure=False)

    objects = client.list_objects(checkpoint_bucket)
    asset_name = output_filename
    exists1 = False
    for obj in objects:
        if asset_name == obj.object_name:
            exists1 = True
            client.fget_object(checkpoint_bucket, asset_name, asset_name)
            break

    objects = client.list_objects(checkpoint_bucket)
    asset_name = metadata_filename
    exists2 = False
    for obj in objects:
        if asset_name == obj.object_name:
            exists2 = True
            client.fget_object(checkpoint_bucket, asset_name, asset_name)
            break

    exists = exists1 and exists
if not exists:
    git.Git(".").clone("https://github.com/" +
                       "ieee8023/covid-chestxray-dataset.git")
    shutil.move("covid-chestxray-dataset/images", ".")
    shutil.move("covid-chestxray-dataset/metadata.csv", ".")
    shutil.rmtree('covid-chestxray-dataset')
    zipdir('images.zip', 'images')
if not exists:
    size = os.path.getsize(output_filename)
    with open(output_filename, 'rb') as fh:
        buf = BytesIO(fh.read())
        result = client.put_object(
            checkpoint_bucket, output_filename, buf, length=size
        )

    size = os.path.getsize(metadata_filename)
    with open(metadata_filename, 'rb') as fh:
        buf = BytesIO(fh.read())
        result = client.put_object(
            checkpoint_bucket, metadata_filename, buf, length=size
        )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10628366247755835, 'fraction_numerical': 0.008258527827648115, 'mean_word_length': 3.4082278481012658, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '447857', 'license': 'Apache-2.0', 'n_tokens_mistral': 878, 'n_tokens_neox': 861, 'n_words': 179, 'path': 'input/input-covid-chestxray.ipynb', 'repo_name': 'Tales-Carvalho/component-library', 'size': '2783'}