<jupyter_start><jupyter_text>1. Setup<jupyter_code>import sys
sys.path.append('../..')
import config
import matplotlib.pyplot as plt
import numpy as np
import os
import warnings

from annotations import *
from density_maps import create_and_save_density_maps
from utils.data.data_ops import move_val_split_to_train
from utils.input_output.io import save_np_arrays, load_np_arrays, load_images
from utils.input_output.io import save_gt_counts, load_gt_counts
from utils.preprocessing.misc import gaussian_smoothing
%matplotlib inline
%load_ext autoreload
%autoreload 2

warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>2. Datasets  2.1 VGG Cells Dataset<jupyter_code>DATASET_PATH = '../../datasets/vgg_cells'
TRAIN_PATH = f'{DATASET_PATH}/train'
TRAIN_IMG_PATH = f'{TRAIN_PATH}/images'
TRAIN_GT_DOTS_PATH = f'{TRAIN_PATH}/gt_dots'
TRAIN_GT_COUNTS_PATH = f'{TRAIN_PATH}/gt_counts'
TRAIN_GT_DENSITY_MAPS_PATH = f'{TRAIN_PATH}/gt_density_maps'

VAL_PATH = f'{DATASET_PATH}/val'

TEST_PATH = f'{DATASET_PATH}/test'
TEST_IMG_PATH = f'{TEST_PATH}/images'
TEST_GT_DOTS_PATH = f'{TEST_PATH}/gt_dots'
TEST_GT_COUNTS_PATH = f'{TEST_PATH}/gt_counts'
TEST_GT_DENSITY_MAPS_PATH = f'{TEST_PATH}/gt_density_maps'
move_val_split_to_train(VAL_PATH, TRAIN_PATH)
!rm -rf $TRAIN_GT_DENSITY_MAPS_PATH
!rm -rf $TRAIN_GT_COUNTS_PATH
!rm -rf $TEST_GT_DENSITY_MAPS_PATH
!rm -rf $TEST_GT_COUNTS_PATH

!mkdir $TRAIN_GT_DENSITY_MAPS_PATH
!mkdir $TRAIN_GT_COUNTS_PATH
!mkdir $TEST_GT_DENSITY_MAPS_PATH
!mkdir $TEST_GT_COUNTS_PATH
print(DATASET_PATH)
print(os.listdir(DATASET_PATH))
print(TRAIN_PATH)
print(os.listdir(TRAIN_PATH))
train_img_names = sorted(os.listdir(TRAIN_IMG_PATH))
train_dots_names = sorted(os.listdir(TRAIN_GT_DOTS_PATH))
test_img_names = sorted(os.listdir(TEST_IMG_PATH))
test_dots_names = sorted(os.listdir(TEST_GT_DOTS_PATH))

print(f'train split: {len(train_img_names)} images')
print(train_img_names[:3])
print(train_dots_names[:3])

print(f'\ntest split: {len(test_img_names)} images')
print(test_img_names[:3])
print(test_dots_names[:3])
train_dots_names = sorted(os.listdir(TRAIN_GT_DOTS_PATH))
test_dots_names = sorted(os.listdir(TEST_GT_DOTS_PATH))

print(TRAIN_GT_DOTS_PATH)
print(train_dots_names[:5])

print(TEST_GT_DOTS_PATH)
print(test_dots_names[:5])<jupyter_output>../../datasets/vgg_cells/train/gt_dots
['001dots.png', '002dots.png', '003dots.png', '004dots.png', '005dots.png']
../../datasets/vgg_cells/test/gt_dots
['101dots.png', '102dots.png', '103dots.png', '104dots.png', '105dots.png']<jupyter_text>Load dots images (.png)<jupyter_code>train_dots_images = load_dots_images(TRAIN_GT_DOTS_PATH, train_dots_names)
test_dots_images = load_dots_images(TEST_GT_DOTS_PATH, test_dots_names)

print(len(train_dots_images), train_dots_images[0].shape, train_dots_images[0].dtype,
      train_dots_images[0].min(), train_dots_images[0].max(), train_dots_images[0].sum())<jupyter_output>100 (256, 256) float64 0.0 1.0 135.0<jupyter_text>Save gt counts (from dots images)<jupyter_code>train_counts = dots_images_to_counts(train_dots_images)
test_counts = dots_images_to_counts(test_dots_images)
save_gt_counts(train_counts, train_dots_names, TRAIN_GT_COUNTS_PATH)
save_gt_counts(test_counts, test_dots_names, TEST_GT_COUNTS_PATH)<jupyter_output><empty_output><jupyter_text>Create and save density maps (.npy)<jupyter_code>create_and_save_density_maps(train_dots_images, config.VGG_CELLS_SIGMA,
                             train_img_names, TRAIN_GT_DENSITY_MAPS_PATH)
create_and_save_density_maps(test_dots_images, config.VGG_CELLS_SIGMA,
                             test_img_names, TEST_GT_DENSITY_MAPS_PATH)<jupyter_output><empty_output><jupyter_text>Load some train images and density maps<jupyter_code>train_images = load_images(TRAIN_IMG_PATH, train_img_names, num_images=3)
print(len(train_images))
print(train_images[0].dtype)
train_gt_density_maps = load_np_arrays(TRAIN_GT_DENSITY_MAPS_PATH, num=3)
print(len(train_gt_density_maps))
print(train_gt_density_maps.dtype)
NUM_PLOTS = 3
plt.figure(figsize=(15, 13))

for i in range(NUM_PLOTS):
    count = train_dots_images[i].sum().astype(np.int)
    
    plt.subplot(3, NUM_PLOTS, i + 1)
    plt.title(f'Initial image (GT_dots count: {count})')
    plt.imshow(train_images[i])
    
    plt.subplot(3, NUM_PLOTS, NUM_PLOTS + i + 1)
    plt.title(f'GT_dots count: {count}')
    plt.imshow(train_dots_images[i], cmap='gray')
    
    plt.subplot(3, NUM_PLOTS, 2 * NUM_PLOTS + i + 1)
    plt.title(f'GT_dots: {count}, GT_density_map: {train_gt_density_maps[i].sum():.2f}')
    plt.imshow(train_gt_density_maps[i], cmap='jet')
    plt.colorbar()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11877147766323023, 'fraction_numerical': 0.017396907216494846, 'mean_word_length': 7.85361216730038, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '504389', 'license': 'MIT', 'n_tokens_mistral': 1966, 'n_tokens_neox': 1906, 'n_words': 249, 'path': 'utils/data/density_map_generation_vgg_cells.ipynb', 'repo_name': 'Tudor67/Object-Counting', 'size': '4656'}