<jupyter_start><jupyter_text>There are some Excel spreadsheet with species labels, but the labels weren't made with the path to the image in the entry but just the location and timestamp, so too hard to recover. So here, we just try getting a list of all species, and then if the species name is in the path to an image, that image will be labeled with that species.  Get available species labels<jupyter_code>with open('/Users/siyuyang/Source/temp_data/CameraTrap/engagements/BNF/20190624and0815/8714_images.json') as f:
    image_paths_original = json.load(f)
image_paths_original = [i for i in image_paths_original if i.lower().endswith('.jpg')]
len(image_paths_original)
sample(image_paths_original, 5)
image_paths = [i.lower() for i in image_paths_original if i.lower().endswith('.jpg')]
len(image_paths)
sample(image_paths, 5)<jupyter_output><empty_output><jupyter_text>Getting a list of speciesOnly ran once to grab the majority of species.<jupyter_code>folder_images = []
for p in image_paths:
    if p.startswith('20190624cameratraps/images/kutai 1'):
        folder_images.append(p)
len(folder_images)
species_sample = set()

for i in folder_images:
    species_sample.add(i.split('/')[-2])
species_sample
bawan_images<jupyter_output><empty_output><jupyter_text>Manually compiled list of species<jupyter_code># this list includes all the species names and typo variants I was able to find best-effort
# can't use rat, cat, bat - these match too many strings

species = ['nothing', 'bird', 'argus', 'crested fireback', 'grey headed fish eagle', 
         'bornean ground cuckoo', 'white headed fish eagle', 'babbler brown bird',
           'camera set-up', 'humans', 'human', 'camera placing shots',
           'civet', 'common palm civet', 'clouded leopard', 'deer', 'mongooe', 'pangolin', 'pig', 'porcupine', 'primate', 'sun bear', 
           'unclear', 'unknown',
          'banded palm civet', 'bay cat', 'bearded pig', 'binturong', 'gibbon', 'great argus', 'leopard cat', 'long-tailed macaque',
          'malay civet', 'marbled cat', 'moon rat', 'mouse deer', 'muntjac', 'orangutan', 'orang-utans', 'pig-tailed macaque', 'red langur', 'sambar deer',
          'banded palm civet', 'banteng', 'bornean red muntja', 'macaque', 'squirrel',
          'turtle', 'banded linsang', 'monitor lizard', 'flat-headed-cat', 'lizard', 'yellow muntjac',
          'small toothed palm civet', 'st palm civet', 'small thoodhed palm civet', 'small thoothed palm civet', 'small thoodheed palm civet', 
           'small-thoodhed palm civet', 'small-toodhed palm civet', 'small-toothed palm civet',
           "storm's stork", "strom's strok", 'storms stork', 'storm stork', 'storms st', 'raptor',
          'rodent', 'reptile', 'orang utan', 'yellow throated marten', 'bats', 
           'collard mongoose', 'collared mongoose',
          'brown wood owl', 'fairy pitta', 'pig-t-macaque',
          'sunbear', 'eagle', 
           'short tailed mongoose', 'short-t-mongoose', 'short-t mongoose', 'short tailed-mongoose', 'short taled-mongoose', 's-t mongoose', 'short-tailed-mongoose', 'short - tailed mongoose', 'short tailet mongoose', 'short-tailed mongoose', 
           'mongoose', 
           'treeshrew', 'marble cat', 'flat headed cat',
          'hunter dog', 'dog hunter', 'short-t-monggoose',
          'reed-leaf monkey', 'red leaf monkey', 'monkey', 'otters civet', 'otter civet', 'butterfly'] # there are no "empty" in the file names anywhere, and there are very few "nothing"
species = set(species)
len(species)
# the dict in this direction is for verification only 

species_to_image_paths = defaultdict(list)

for p in tqdm(image_paths_original):
    candidate_species = []
    for s in species:
        if s in p.lower():
            candidate_species.append(s)
            
    if len(candidate_species) == 0:
        continue
    
    if 'nothing' in candidate_species:
        species_to_image_paths['nothing'].append(p)
    
    most_specific_species = None
    max_len = 0
    for c in candidate_species:
        if len(c) > max_len:
            max_len = len(c)
            most_specific_species = c
            
    species_to_image_paths[most_specific_species].append(p)
len(species_to_image_paths.keys())
len(species_to_image_paths['nothing']) # too many wrongly spelt short-tailed mongoose... They'll be in "mongoose"
'20190624cameratraps/images/OUTROP CAMERA TRAPS ALL/CAMERA TRAPS SABANGAU/CAMERA TRAPS 2009/other stuff/CAMERA TRAPS 2009 up to date/KM 2 x Railway/Km2 x Railway Download 23-05-09 NOTHING macaque only/CDY_0031.JPG' in species_to_image_paths['nothing']
image_path_to_species = {}

for s, list_paths in species_to_image_paths.items():
    for p in list_paths:
        image_path_to_species[p] = s
len(image_path_to_species)  # about 1k are "nothing" labeled
sample(image_path_to_species.items(), 5)<jupyter_output><empty_output><jupyter_text>Make CCT database<jupyter_code>info = {
  # Required
  "version" : '20190825',
  "description" : 'Images sent by <NAME> from Borneo Nature Foundation',
  
  # Optional
  "year" : 2019,
  "contributor" : 'Images sent by <NAME> from Borneo Nature Foundation. DB created by <NAME>',
  "date_created" : str(datetime.today().date())
}
info
len(species_to_image_paths)
sorted(species_to_image_paths.keys())
cat_map = {
    'nothing': 'empty',
    "strom's strok": "storm's stork",
    "storms stork": "storm's stork",
    "storms st": "storm's stork",
    "storm stork": "storm's stork", 
    'pig-t-macaque': 'pig-tailed macaque',
    "unclear": "unknown",
    'camera set-up': 'human',
    'humans': 'human',
    'orang-utans': 'orangutan',
    'orang utan': 'orangutan',
    'sunbear': 'sun bear',
    'marble cat': 'marbled cat',
    'flat-headed-cat': 'flat headed cat',
    'dog hunter': 'hunter dog',
    'small thoodhed palm civet': 'small toothed palm civet',
    'short tailed-mongoose': 'short-tailed mongoose',
    'short taled-mongoose': 'short-tailed mongoose',
    'short-t mongoose': 'short-tailed mongoose',
    'short-t-monggoose': 'short-tailed mongoose',
    'short tailed mongoose': 'short-tailed mongoose',
    'short-t-mongoose': 'short-tailed mongoose',
    'short-tailed-mongoose': 'short-tailed mongoose',
    's-t mongoose': 'short-tailed mongoose',
    'short - tailed mongoose': 'short-tailed mongoose',
    'short tailet mongoose': 'short-tailed mongoose',
    'mongooe': 'mongoose',
    'reed-leaf monkey': 'red leaf monkey',
    'otters civet': 'otter civet',
    'small toothed palm civet': 'small-toothed palm civet', 
    'st palm civet': 'small-toothed palm civet', 
    'small thoodhed palm civet': 'small-toothed palm civet', 
    'small thoothed palm civet': 'small-toothed palm civet', 
    'small thoodheed palm civet': 'small-toothed palm civet',
    'small-thoodhed palm civet': 'small-toothed palm civet',
    'small-toodhed palm civet': 'small-toothed palm civet',
    'collard mongoose': 'collared mongoose'
}
# redo image_path_to_species to uniformize species names
for p, s in image_path_to_species.items():
    if s in cat_map:
        image_path_to_species[p] = cat_map[s]

len(image_path_to_species)

valid_species = set(image_path_to_species.values())
len(valid_species)
valid_species
categories = {}
i = 1
for s in sorted(list(valid_species)):
    categories[s] = i
    i += 1
images = []
annotations = []

for p, s in image_path_to_species.items():
    if s in cat_map:
        s = cat_map[s]
        
    image_id = str(uuid.uuid4())
    anno_id = str(uuid.uuid4())
    category_id = categories[s]
    
    images.append({
        'id': image_id,
        'file_name': p
    })
    annotations.append({
        'id': anno_id,
        'image_id': image_id,
        'category_id': category_id
    })
len(images)
len(annotations)
images[100]
annotations[100]
categories_final = []
for name, i in categories.items():
    categories_final.append({
        'id': i,
        'name': name
    })
cct_db = {
    'info': info,
    'images': images,
    'annotations': annotations,
    'categories': categories_final
}
with open('/Users/siyuyang/Source/temp_data/CameraTrap/engagements/BNF/20190624and0815/BNF_20190624and0815_20190825a.json', 'w') as f:
    json.dump(cct_db, f, indent=1)<jupyter_output><empty_output><jupyter_text>Split detector results into labeled and unlabeled portionsThere was no reliable labels for "empty" images.<jupyter_code>with open('/Users/siyuyang/Source/temp_data/CameraTrap/engagements/BNF/20190624and0815/8714_detections_bnf0624and0815_20190815182245_refiltered.json') as f:
    all_res = json.load(f)
len(all_res['images'])
images_with_label = set(image_path_to_species.keys())
len(images_with_label)
labeled_im_res = []
unlabeled_im_res = []

for res in all_res['images']:
    if res['file'] in images_with_label:
        labeled_im_res.append(res)
    else:
        unlabeled_im_res.append(res)
len(labeled_im_res)
len(unlabeled_im_res)
all_res.keys()
labeled_res = {
    'info': all_res['info'],
    'detection_categories': all_res['detection_categories'],
    'images': labeled_im_res
}

unlabeled_im_res = {
    'info': all_res['info'],
    'detection_categories': all_res['detection_categories'],
    'images': unlabeled_im_res
}
with open('/Users/siyuyang/Source/temp_data/CameraTrap/engagements/BNF/20190624and0815/8714_detections_bnf0624and0815_20190815182245_refiltered_labeled.json', 'w') as f:
    json.dump(labeled_res, f, indent=1)
with open('/Users/siyuyang/Source/temp_data/CameraTrap/engagements/BNF/20190624and0815/8714_detections_bnf0624and0815_20190815182245_refiltered_unlabeled.json', 'w') as f:
    json.dump(unlabeled_im_res, f, indent=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1287313432835821, 'fraction_numerical': 0.02394278606965174, 'mean_word_length': 4.146133333333333, 'pattern_counts': {'":': 5, '<': 24, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '337910', 'license': 'MIT', 'n_tokens_mistral': 3573, 'n_tokens_neox': 3341, 'n_words': 836, 'path': 'data_management/importers/bnf_to_json.ipynb', 'repo_name': 'col-kol/CameraTraps', 'size': '9663'}