<jupyter_start><jupyter_text>Load hierarchy<jupyter_code>NAME = {} # wordnetid -> name
with open('node_names.txt') as f:
    for l in f.readlines():
        wnid, text = l.strip().split('\t')
        NAME[wnid] = text
children = {wnid: [] for wnid in NAME}

with open('class_hierarchy.txt') as f:
    for l in f.readlines():
        parent, child = l.strip().split()
        children[parent].append(child)<jupyter_output><empty_output><jupyter_text>Create top-down structure<jupyter_code>root = 'n00001740' # entity 

levels = []
next_level = [root]

while next_level:
    levels.append(next_level)
    current_level = next_level
    next_level = []
    for node in current_level:
        for child in children[node]:
            next_level.append(child)
def leaves_reachable(node):
    leaves = []
    todo = [node]
    while todo:
        curr = todo
        todo = []
        for child in curr:
            if not children[child]:
                leaves.append(child)
            else:
                for c in children[child]:
                    todo.append(c)
    return leaves

def is_child(child, parent):
    if child in children[parent]:
        return True
    
    return any(is_child(child, n) for n in children[parent])
datasets = {
    'Entity13':    (3, 'n00001740', 20),
    'Entity30':    (4, 'n00001740',  8),
    'Living17':    (5, 'n00004258',  4),
    'NonLiving26': (5, 'n00021939',  4)
}
    
DATASET = 'Living17'
    
level, root, num_classes = datasets[DATASET]

nodes = sorted(levels[level])
rng = np.random.RandomState(seed=2)
for n in nodes:
    leaves = list(sorted(leaves_reachable(n)))
    if len(leaves) < num_classes or not is_child(n, root): continue
    print(NAME[n])
    subclasses = list(rng.choice(leaves, num_classes, replace=False))
    per_class = num_classes // 2
    src, tgt = subclasses[:per_class], subclasses[per_class:]
    for d, t in [(src, 'Source'), (tgt, 'Target')]:
        print(f' [{t}]')
        for c in d:
            print(f' -- {NAME[c]}')<jupyter_output>salamander
 [Source]
 -- eft
 -- axolotl, mud puppy, Ambystoma mexicanum
 [Target]
 -- common newt, Triturus vulgaris
 -- spotted salamander, Ambystoma maculatum
turtle
 [Source]
 -- box turtle, box tortoise
 -- leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea
 [Target]
 -- loggerhead, loggerhead turtle, Caretta caretta
 -- mud turtle
lizard
 [Source]
 -- whiptail, whiptail lizard
 -- alligator lizard
 [Target]
 -- African chameleon, Chamaeleo chamaeleon
 -- banded gecko
snake, serpent, ophidian
 [Source]
 -- night snake, Hypsiglena torquata
 -- garter snake, grass snake
 [Target]
 -- sea snake
 -- boa constrictor, Constrictor constrictor
spider
 [Source]
 -- tarantula
 -- black and gold garden spider, Argiope aurantia
 [Target]
 -- garden spider, Aranea diademata
 -- wolf spider, hunting spider
grouse
 [Source]
 -- ptarmigan
 -- prairie chicken, prairie grouse, prairie fowl
 [Target]
 -- ruffed grouse, partridge, Bonasa umbellus
 -- black grouse
parrot
 [Source]
[...]
{'fraction_non_alphanumeric': 0.12603374131657294, 'fraction_numerical': 0.020178630499503806, 'mean_word_length': 2.968503937007874, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '627909', 'license': 'MIT', 'n_tokens_mistral': 1102, 'n_tokens_neox': 1029, 'n_words': 276, 'path': 'hierarchy.ipynb', 'repo_name': 'anonymousterrier/BREEDS-Benchmarks', 'size': '3023'}