<jupyter_start><jupyter_text>Make a 6-Metanode-network<jupyter_code>import sys
sys.path.append('../../hetnet-ml/src')

import pandas as pd
import graph_tools as gt
from hetio.hetnet import MetaGraph
def num_metapaths(nodes, edges):
    abv, met = gt.get_abbrev_dict_and_edge_tuples(gt.add_colons(nodes), gt.add_colons(edges))
    return len(MetaGraph.from_edge_tuples(met, abv).extract_metapaths('Chemicals & Drugs', 'Disorders', 4))
nodes = gt.remove_colons(pd.read_csv('../data/nodes_VER31_R_consolidated_condensed_filtered_001.csv'))
edges = gt.remove_colons(pd.read_csv('../data/edges_VER31_R_consolidated_condensed_filtered_001.csv'))
print('{:,} Potnential Metapaths'.format(num_metapaths(nodes, edges)))<jupyter_output>8,745 Potnential Metapaths<jupyter_text>Some nodes really have little to do with bio-medical knowledge and these will also be removed.- REMOVE NODES:    - Organizations    - Activities & Behaviors    - Concepts & Ideas    - Procedures    - Devices    - Living BeingsNOTE: While `Living Beings` seems like it would be useful, many of these are far too general, making for replationships and paths that are not meaningful (`Researcher - Asscoided With - Amino Acids` for example).  It is too difficult to filter out the general terms from the more specific, so all Living Beings will be removed  Removal of un-needed metanodes<jupyter_code># Remove nodes of types that are less-useful
remove_types = ['Organizations', 'Activities & Behaviors', 'Concepts & Ideas', 'Procedures', 'Devices', 'Living Beings']
idx = gt.remove_colons(nodes).query('label in @remove_types').index
nodes.drop(idx, inplace=True)
# Make sure that 
ok_ids = nodes['id'].unique()
edges = edges.query('start_id in @ok_ids and end_id in @ok_ids')

print('{:,} Unique IDs in the nodes'.format(len(ok_ids)))
print('{:,} Unique IDs found wihtin the remaining edges'.format(len(set(edges['start_id']).union(set(edges['end_id'])))))
ok_ids = list(set(edges['start_id']).union(set(edges['end_id'])))
nodes = nodes.query('id in @ok_ids')

print('Remvoed IDs from nodes that no longer have edges...')
print('{:,} IDs found in edges'.format(len(ok_ids)))
print('{:,} IDs found in nodes'.format(len(set(edges['start_id']).union(set(edges['end_id'])))))
print('After cutting down to 6 Most Revelent Metanodes... ')
print('{:,} Potnential Metapaths'.format(num_metapaths(nodes, edges)))
nodes['label'].value_counts()
edges['type'].nunique()
edges['type'].value_counts()
gt.add_colons(nodes).to_csv('../data/nodes_VER31_R_cons_6_metanode.csv', index=False)
gt.add_colons(edges).to_csv('../data/edges_VER31_R_cons_6_metanode.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14210327930644553, 'fraction_numerical': 0.008669430833019224, 'mean_word_length': 6.41340782122905, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '392777', 'license': 'PostgreSQL', 'n_tokens_mistral': 955, 'n_tokens_neox': 895, 'n_words': 217, 'path': '1_build/05-keep_6_relvent_metanodes.ipynb', 'repo_name': 'turoger/semmed', 'size': '2653'}