<jupyter_start><jupyter_text>Data preparation 1Useful resources:* Used dataset:    * [NIH Clinical Center provides one of the largest publicly available chest x-ray datasets to scientific community](https://www.nih.gov/news-events/news-releases/nih-clinical-center-provides-one-largest-publicly-available-chest-x-ray-datasets-scientific-community)    * [NIH CC ChestXRay Dataset (box.com)](https://nihcc.app.box.com/v/ChestXray-NIHCC)    * [NIH CC ChestXRay Dataset (Google Cloud)](https://cloud.google.com/healthcare/docs/resources/public-datasets/nih-chest)* Datasets:    * [Google Dataset Search](https://datasetsearch.research.google.com/)    * [Awesome Public Datsets on GitHub](https://github.com/awesomedata/awesome-public-datasets)* Pandas:    * [10 minutes to Pandas tutorial](https://pandas.pydata.org/pandas-docs/stable/user_guide/10min.html)  Imports<jupyter_code>import os
import urllib.request
from multiprocessing import Pool, cpu_count

import pandas as pd
import tqdm
from PIL import Image<jupyter_output><empty_output><jupyter_text>Constants<jupyter_code>#  _______________________________________
# / Set 'DOWNLOAD_IMAGES' to True only if \
# \ time you have.                        /
#  ---------------------------------------
#    \
#     \        .
#      .---.  //
#     Y|o o|Y//
#    /_(i=i)K/
#    ~()~*~()~
#     (_)-(_)

#      Darth
#      Vader
#      koala
DOWNLOAD_IMAGES = False

DATA_DIR = 'data/'
IMAGES_DIR = os.path.join(DATA_DIR, 'images')
IMAGE_NEW_RESOLUTION = (256, 256)
ORIGINAL_CSV_FILE = os.path.join(DATA_DIR, 'Data_Entry_2017_v2020.csv')

# Links to archives containing original NIH Clinical Center ChestX-ray8 dataset
LINKS = [
    'https://nihcc.box.com/shared/static/vfk49d74nhbxq3nqjg0900w5nvkorp5c.gz',
    'https://nihcc.box.com/shared/static/i28rlmbvmfjbl8p2n3ril0pptcmcu9d1.gz',
    'https://nihcc.box.com/shared/static/f1t00wrtdk94satdfb9olcolqx20z2jp.gz',
    'https://nihcc.box.com/shared/static/0aowwzs5lhjrceb3qp67ahp0rd1l1etg.gz',
    'https://nihcc.box.com/shared/static/v5e3goj22zr6h8tzualxfsqlqaygfbsn.gz',
    'https://nihcc.box.com/shared/static/asi7ikud9jwnkrnkj99jnpfkjdes7l6l.gz',
    'https://nihcc.box.com/shared/static/jn1b4mw4n6lnh74ovmcjb8y48h8xj07n.gz',
    'https://nihcc.box.com/shared/static/tvpxmn7qyrgl0w8wfh9kqfjskv6nmm1j.gz',
    'https://nihcc.box.com/shared/static/upyy3ml7qdumlgk2rfcvlb9k6gvqq2pj.gz',
    'https://nihcc.box.com/shared/static/l6nilvfa9cg3s28tqv1qc1olm3gnz54p.gz',
    'https://nihcc.box.com/shared/static/hhq8fkdgvcari67vfhs7ppg2w6ni4jze.gz',
    'https://nihcc.box.com/shared/static/ioqwiy20ihqwyr8pf4c24eazhh281pbu.gz'
]<jupyter_output><empty_output><jupyter_text>Download and unpack the dataset<jupyter_code>def download(idx_link):
    idx, link = idx_link
    fn = 'images_%02d.tar.gz' % (idx+1)
    if os.path.exists(os.path.join(DATA_DIR, fn)):
        print('File {} already exists. Skipping...'.format(fn))
        return None
    print('downloading {}...'.format(fn))
    urllib.request.urlretrieve(link, os.path.join(DATA_DIR, fn))
print('Using {} threads'.format(cpu_count()))

os.makedirs(DATA_DIR, exist_ok=True)

if DOWNLOAD_IMAGES:
    with Pool(cpu_count()) as p:
        r = list(
            tqdm.tqdm(
                p.imap(download, zip(list(range(len(LINKS))), LINKS)), 
                total=len(LINKS)
            )
        )
if DOWNLOAD_IMAGES:
    archives = [os.path.join(DATA_DIR, archive) for archive in os.listdir(DATA_DIR) if archive.startswith('images_') and archive.endswith('.tar.gz')]

    if not os.path.exists(IMAGES_DIR):
        pbar = tqdm.tqdm(archives)
        for archive in pbar:
            pbar.set_description('Unpacking {}'.format(archive), refresh=True)
            ! tar -xzf $archive -C $DATA_DIR
    else:
        print('{} exists, skipping unpacking'.format(IMAGES_DIR))
if not os.path.exists(ORIGINAL_CSV_FILE):
    ! gdown --id 1FVTrdLVjPvJKtU60a75Z119aABOy-YbB -O $ORIGINAL_CSV_FILE
else:
    print('Original CSV file ({}) already exists.'.format(ORIGINAL_CSV_FILE))<jupyter_output><empty_output><jupyter_text>Explore the datasetTasks:1. Load the CSV file and display its content.1. Explore the dataset. Check the `Finding Labels` column.<jupyter_code>## Load the CSV file and display its content

## Explore the dataset. Check the Finding Labels column.<jupyter_output><empty_output><jupyter_text>Decrease number of samplesTasks:1. Remove samples with multiple labels (hint: multiple labels are divided with |)1. Remove samples with labels from "V2" version of the dataset. **Keep the `No Finding` category!**1. Randomly drop 50.000 samples with `No Finding` label  Remove samples with multiple labels (hint: multiple labels are divided with |)<jupyter_code>## Remove samples with multiple labels (hint: multiple labels are divided with |)<jupyter_output><empty_output><jupyter_text>Remove samples with `findings_v2` labels<jupyter_code>FINDINGS_V1 = ['Atelectasis', 'Cardiomegaly', 'Effusion', 'Infiltration', 'Mass', 'Nodule', 'Pneumonia', 'Pneumothorax']
FINDINGS_V2 = ['Hernia', 'Emphysema', 'Pleural_Thickening', 'Fibrosis', 'Consolidation', 'Edema']
              # We will keep 'No Finding' class of the 'v2' dataset for this workshop
## Remove samples with labels from "V2" version of the dataset. Keep the No Finding category!<jupyter_output><empty_output><jupyter_text>Randomly drop 50.000 samples with `No Finding` label<jupyter_code>## Randomly drop 50.000 samples with `No Finding` label<jupyter_output><empty_output><jupyter_text>Sync DF with images<jupyter_code>def sync_files_csv():
    images = {f for f in os.listdir(IMAGES_DIR) if f.endswith('.png')}
    image_index = set(df['Image Index'])
    print('sync_files_csv - Before images: {} ; Before CSV: {}'.format(len(images), len(df)))
    
    diff = list(images.symmetric_difference(image_index))
    print('images - image_index: {}'.format(len(diff)))
    
    for image in tqdm.tqdm(diff):
        try:
            os.remove(os.path.join(IMAGES_DIR, image))
        except:
            pass
        df.drop(df[df['Image Index'].str.match(image)].index, inplace=True)
    
    images = {f for f in os.listdir(IMAGES_DIR) if f.endswith('.png')}
    print('sync_files_csv - After images: {} ; After CSV: {}'.format(len(images), len(df)))

if DOWNLOAD_IMAGES:
    sync_files_csv()<jupyter_output><empty_output><jupyter_text>Resize images<jupyter_code>def resize(img_path):
    try:
        img = Image.open(img_path)
        img_resized = img.resize(IMAGE_NEW_RESOLUTION)
        img_resized.save(img_path, "PNG")
    except Exception as ex:
        print('Error when processing {}. Removing image. Exception: {}'.format(img_name, ex))
        os.remove(img_path)
if DOWNLOAD_IMAGES:
    images = [os.path.join(IMAGES_DIR, img_name) for img_name in os.listdir(IMAGES_DIR) if img_name.endswith('.png')]

    print('Resizing images to {}'.format(IMAGE_NEW_RESOLUTION))

    with Pool(cpu_count()) as p:
        r = list(
            tqdm.tqdm(
                p.imap(resize, images),
                total=len(images)
            )
        )<jupyter_output><empty_output><jupyter_text>Save results<jupyter_code>if DOWNLOAD_IMAGES:
    archive = os.path.join(DATA_DIR, 'nih_chest_xray_single_9c_{}x{}.tar.gz'.format(IMAGE_NEW_RESOLUTION[0], IMAGE_NEW_RESOLUTION[1]))

    if not os.path.exists(archive):
        ! tar -czf $archive $IMAGES_DIR
    else:
        print('Archive {} already exists'.format(archive))
    
    df.reindex()
    df.to_csv(os.path.join(DATA_DIR, 'nih_chest_xray_single_9c.csv'))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13654405927494045, 'fraction_numerical': 0.02103731145805769, 'mean_word_length': 3.9340731070496084, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 41, 'https://': 18, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '160356', 'license': 'Apache-2.0', 'n_tokens_mistral': 2873, 'n_tokens_neox': 2727, 'n_words': 493, 'path': '0_data_preparation_1.ipynb', 'repo_name': 'issdl/from-data-to-solution-2021', 'size': '7558'}