<jupyter_start><jupyter_text>Background* AIM Convert dend.json to:   * A set of OWL individuals linked by subClusterOf relationships.   * Links from OWL Individuals to functions (and morphologies?)   * A (more limited) set of OWL Classes whose classificatin is driven by subclustering & links from Individuals to functions/morphologies - see   * Test data in this notebook comes from Allen consensus clustering of MTG cells provided by <NAME>.  Here is a visualisation of the dendrogram:![image](https://user-images.githubusercontent.com/112839/95180282-abf16400-07b9-11eb-9b8d-9f84a7abf757.png)<jupyter_code>import json
f = open('dend.json', 'r')
j = json.loads(f.read())
j.keys()


# Bifurcation of tree at root

len(j['children'])
# Inspect  root node:

j['node_attributes']
# A simple transformation to a list of nodes and edges for easier processing (Ultimately this should be JSON-LD)

import warnings

out = {}

def tree_recurse(tree, out, parent_node_id = ''):
    """Convert Allen Taxonomy JSON to a list of nodes and edges, where nodes are
    Copies of nodes in Allen JSON & edges are duples - (subject(child), object(parent))
    identified by 'cell_set_accession'.
    
    Args:
        - Tree: Allen taxonomy in JSON, or some subtree of it
        - Output structure to populate (starting point must be an empty dict)
        - parent_node_id: 'cell_set_accession' of parent.  Default (no parent) = root.
        
    """
    if not out:
        out['nodes']= []
        out['edges'] = set()
    if 'node_attributes' in tree.keys():
        if len(tree['node_attributes']) > 1:
            warnings.warn("Don't know how to deal with multiple nodes per recurse")
        ID = tree['node_attributes'][0]['cell_set_accession']
            
        out['nodes'].append(tree['node_attributes'][0])
        if parent_node_id:
            out['edges'].add((ID, parent_node_id))
        if 'children' in tree.keys():
            for c in tree['children']:
                tree_recurse(c, out, parent_node_id = ID)
        else:
            warnings.warn("non leaf node %s has no children" % ID)
    elif 'leaf_attributes' in tree.keys():
        if len(tree['leaf_attributes']) > 1:
            warnings.warn("Don't know how to deal with multiple nodes per recurse")       
        ID = tree['leaf_attributes'][0]['cell_set_accession']
        # Tag leaves
        tree['leaf_attributes'][0]['is_leaf'] = True
        out['nodes'].append(tree['leaf_attributes'][0])
        out['edges'].add((ID, parent_node_id))
        if 'children' in tree.keys():
            warnings.warn('leaf node %s has children!' % ID)
    else:
        warnings.warn("No recognized nodes")
        
tree_recurse(j, out)
for n in out['nodes']
list(out['edges'])[0:10]
 '|'.join([e[1] for e in out['edges'] if e[0] == 'CS201908210_104'])
# view paths to root (single inheritance assumed):

def ascend(node_id, edges, path):
    p = [e[1] for e in edges if e[0] == node_id]
    if not path:
        path.append(node_id)
    if p:
        path.append(p[0])
        ascend(p[0], edges, path)
        
def id_conv(path, nodes):
    out = []
    for p in path:
        pref_alias = [n['cell_set_preferred_alias'] for n in nodes 
                      if (n['cell_set_accession'] == p) and n['cell_set_preferred_alias'] ]
        if pref_alias:
            out.extend(pref_alias)
        else:
            out.append(p)
    return out

for n in out['nodes']:
    if 'is_leaf' in n.keys() and n['is_leaf']:
        path_to_root = []
        ascend(node_id = n['cell_set_accession'], edges=out['edges'], path=path_to_root)
        print(id_conv(path_to_root, out['nodes']))
        


## Check for unique attribute values

def unique_attribute_test(nodes):
    # Assuming root node has all keys
    for attr in nodes[0].keys():
        a = {n[attr] for n in out['nodes'] if attr in n.keys()}
        if len(nodes) == len(a):
            print("%s, e.g. '%s' is a unique attribute" % (attr, nodes[1][attr]))
            
unique_attribute_test(out['nodes'])
# Lets go with cell_set_label as rdfs_label for now.  It may be worth inventing some APs for the others...

# Alternative: 
   # Everything with a cell_set_preferred_alias gets the name "$cell_set_preferred_alias - $taxonomy_id. 
   # We make also classes for these
   # The rest we name "$cell_set_label - $taxonomy_id"
   # other identifiers go in synonyms (at least for now)
robot_template_seed = { 'ID': 'ID', 
                       'Label': 'LABEL',
                       'PrefLabel': 'A skos:prefLabel',
                       'Entity Type': 'TI %',
                       'TYPE': 'TYPE',
                       'Property Assertions': "I BDSHELP:subcluster_of SPLIT='|'",
                       'Synonyms': 'A oboInOwl:has_exact_synonym',
                       'Function': 'TI capable_of some %'
                      }


import pandas as pd

robot_template = pd.DataFrame.from_records([robot_template_seed])
robot_template
dl = [robot_template_seed]

synonym_properties = ['original_label', 
                      'cell_set_aligned_alias',
                      'cell_set_additional_aliases']
for o in out['nodes']:
    d = {}
    d['ID'] = 'AllenDend:' + o['cell_set_accession']
    d['TYPE'] = 'owl:NamedIndividual'
    d['Label'] = o['cell_set_label'] + ' - ' + o['cell_set_accession']
    d['PrefLabel'] = o['cell_set_preferred_alias'] + ' - ' + o['cell_set_accession']
    d['Entity Type'] = 'BDSHELP:Cluster'
    d['Synonyms'] = '|'.join([o[prop] for prop in synonym_properties if o[prop]])
    d['Property Assertions'] = '|'.join(['AllenDend:' + e[1] for e in out['edges'] if e[0] == o['cell_set_accession']])
    # There should only be one!
    dl.append(d)

robot_template = pd.DataFrame.from_records(dl)
#print(robot_template[['ID','PrefLabel']][0:50])
#print(robot_template[['ID','PrefLabel']][51:100])
#print(robot_template[['ID','PrefLabel']][100:-1])

robot_template[['ID','PrefLabel']]
    
robot_template.to_csv("CCN201908210.tsv", sep="\t", index=False)<jupyter_output><empty_output><jupyter_text>- [x] Constuct helper files - [x] Class: cell cluster (we have one in VFB but should be generalised - maybe cell set?) - [x] OP subcluster_of - [x] use add-prefix to add curies<jupyter_code>import uuid
'http://brain_data_standards/scratch_' + str(uuid.uuid1())

import pandas as pd
robot_equivalent_class_template_seed = {
                       'ID': 'ID', 
                       'Exemplar': 'EC BDSHELP:has_exemplar value %',
                       'Class_type': 'CLASS_TYPE'
                       #'part of': "SC 'part of' some %"
                       #'located in': "SC 'part of' some %"
                       #'has soma location': "SC 'part of' some %"
                      }

robot_class_curation_seed = {'ID': 'ID',
                             'Label': 'LABEL',
                             'PrefLabel': 'A skos:prefLabel',
                             'Synonyms': 'A oboInOwl:has_exact_synonym',
                             'Exemplar': "EC CL:0000003 and 'has_exemplar' value %",
                             'Classification': 'SC %',
                             'Comment': 'A rdfs:comment',
                             'part of': "SC 'part of' some %",
                             'part evidence comment': '>A rdfs:comment',
                             'part evidence pub': ">A dc:reference SPLIT='|'",  # Bundle this with dbxrefs?
                             'part evidence dbxref': ">A oio:hasDbXref SPLIT='|'",                  
                             'location': "SC 'located in' some %",
                             'location evidence comment': '>A rdfs:comment',
                             'location evidence pub': ">A dc:reference SPLIT='|'",  # Bundle this with dbxrefs?
                             'location evidence dbxref': ">A oio:hasDbXref SPLIT='|'",  
                             'has soma location': "SC 'has soma location' some %",
                             'soma location evidence comment': '>A rdfs:comment',
                             'soma location evidence pub': ">A dc:reference SPLIT='|'",  # Bundle this with dbxrefs?
                             'soma location evidence dbxref': ">A oio:dbxref SPLIT='|'",
                             'cell morphology phenotype': "SC 'bearer of' some %",
                             'soma location evidence comment': '>A rdfs:comment',
                             'soma location evidence pub': ">A dc:reference SPLIT='|'",  # Bundle this with dbxrefs?
                             'soma location evidence dbxref': ">A oio:hasDbXref SPLIT='|'"
                            }


ec_template = [robot_equivalent_class_template_seed]
class_template = [robot_class_curation_seed] 

# not bothering with stable IDs for now:

for o in out['nodes']: 
    ID = 'http://brain_data_standards/scratch_' + str(uuid.uuid1())
    d={}
    d['ID']=ID
    d['Class_type']: 'subclass'
    d['Exemplar']= 'AllenDend:' + o['cell_set_accession']
    if o['cell_set_label']:
        d['Label'] = o['cell_set_label']
    if o['cell_set_preferred_alias']:
        d['PrefLabel'] = o['cell_set_preferred_alias']
    for k in robot_class_curation_seed.keys():
        if not(k in d.keys()):
            d[k] = ''
    class_template.append(d)
    

ec_robot_template = pd.DataFrame.from_records(ec_template)
class_robot_template = pd.DataFrame.from_records(class_template)


#ec_robot_template.to_csv("CCN201908210_EC.tsv", sep="\t", index=False)
class_robot_template.to_csv("CCN201908210_class_curated.tsv", sep="\t", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13185896098476946, 'fraction_numerical': 0.011788024201961194, 'mean_word_length': 2.428826895565093, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 24, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '447992', 'license': 'Apache-2.0', 'n_tokens_mistral': 3125, 'n_tokens_neox': 2984, 'n_words': 778, 'path': 'scratch/.ipynb_checkpoints/allen_taxonomy_hack-checkpoint.ipynb', 'repo_name': 'obophenotype/brain_data_standards_ontologies', 'size': '9593'}