<jupyter_start><jupyter_text>Show Data set size<jupyter_code>import os
import json
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.pylab as pylab
pylab.rcParams['figure.figsize'] = 20, 12

anno_folder = '/data/proj/smFISH/Students/Max_Senftleben/files/annotation/'
print(os.listdir(anno_folder))
def sub(data):
    dicts = {}
    for i in data:
        if i['category_id'] in dicts:
            dicts[i['category_id']] += 1
        else:
            dicts[i['category_id']] = 1
            
    return dicts
            
def show_obj_per_dataset(folder):
    
    for file in os.listdir(folder):
        print(file)
        data = json.load(open(folder + file))
        print(len(data['images']))
        data = data['annotations']
        dicts = sub(data) 
        print(dicts)

# show counts of the images and all the instances in the dataset per class
print('Nuclei____________________')
show_obj_per_dataset(anno_folder + '20190309_aug_pop/')
print('polyt____________________')
show_obj_per_dataset(anno_folder + '20190306_poly_t/')
print('AMEX____________________')
show_obj_per_dataset('/data/proj/smFISH/Students/Max_Senftleben/files/data/20190422_AMEX_transfer_nuclei/annos/')
def show_images(npy_path):
    a  = {}
    for i in os.listdir(npy_path):
        n = np.load(npy_path + i)
        l = np.dsplit(n, n.shape[2])[1:]
        for s in l:
            if np.unique(s)[1] in a:
                a[np.unique(s)[1]] += 1
            else:
                a[np.unique(s)[1]] = 1
    print(a)
show_images('/data/proj/smFISH/Students/Max_Senftleben/files/data/20190422_AMEX_transfer_nuclei/npy/')<jupyter_output>{}
{'fraction_non_alphanumeric': 0.1241631162507608, 'fraction_numerical': 0.027388922702373707, 'mean_word_length': 3.0693069306930694, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '127741', 'license': 'MIT', 'n_tokens_mistral': 623, 'n_tokens_neox': 578, 'n_words': 109, 'path': 'parsing/dataset_size.ipynb', 'repo_name': 'maxsenh/nuclei_cell_detect', 'size': '1643'}