<commit_before>from models import *

import numpy as np
import networkx as nx

class Neuron(object):

    def __init__(self, neuron_id, project_id):

        self.neuron = ClassInstance.objects.get(pk=neuron_id, project_id = project_id)
        self.neuron_id = neuron_id
        self.project_id = project_id

    def get_contained_skeletons(self):

        qs = ClassInstanceClassInstance.objects.filter(
            relation__relation_name='model_of',
            project=self.project_id,
            class_instance_b=self.neuron_id).select_related("class_instance_a__id")

        return [ele.class_instance_a.id for ele in qs]

class Skeleton(object):

    def __init__(self, skeleton_id, project_id = None):

        if project_id is None:
            self.skeleton = ClassInstance.objects.get(pk=skeleton_id)
            project_id = self.skeleton.project_id
        else:
            self.skeleton = ClassInstance.objects.get(pk=skeleton_id, project_id = project_id)

        self.skeleton_id = skeleton_id
        self.project_id = project_id
        self._edge_length_sum = 0.0

        self.graph = self._create_graph()
        self.connected_connectors = self._fetch_connected_connectors()
        self.downstream_skeletons = self._fetch_downstream_skeletons()
        self.upstream_skeletons = self._fetch_upstream_skeletons()

        self._compute_skeleton_edge_deltatime()

        qs = ClassInstanceClassInstance.objects.filter(
            relation__relation_name='model_of',
            project=self.project_id,
            class_instance_a=self.skeleton_id).select_related("class_instance_b")
        self.neuron = qs[0].class_instance_b

    def node_count(self):
        return self.graph.number_of_nodes()

    def input_count(self):
        """ Returns the number of anatomical synaptic inputs onto this Skeleton. """
        n = 0
        for k,v in self.upstream_skeletons.items():
            n += v
        return n

    def output_count(self):
        """ Returns the number of anatomical synaptic outputs onto other Skeleton instances. """
        n = 0
        for k,v in self.downstream_skeletons.items():
            n += v
        return n

    def _fetch_connected_connectors(self):
        """
        Example:
        { connector_id: {'presynaptic_to': [node_id1, node_id2],
                         'postsynaptic_to': [node_id3] }
        """

        qs_tc = TreenodeConnector.objects.filter(
            project=self.project_id,
            skeleton=self.skeleton.id
        ).select_related('connector')

        relations = dict((r.relation_name, r.id) for r in Relation.objects.filter(project=self.project_id))
        results = {}

        for tc in qs_tc:
            if not tc.connector_id in results:
                results[tc.connector_id] = {
                    'presynaptic_to': [],
                    'postsynaptic_to': [],
                    # TODO: labels, location etc.
                }

            if tc.relation_id == relations['presynaptic_to']:
                results[tc.connector_id]['presynaptic_to'].append( tc.treenode_id )
            elif tc.relation_id == relations['postsynaptic_to']:
                results[tc.connector_id]['postsynaptic_to'].append( tc.treenode_id )

        return results

    def _fetch_downstream_skeletons(self):
        """ Returns a list of skeleton IDs that this Skeleton synapses onto. """
        relations = dict((r.relation_name, r.id) for r in Relation.objects.filter(project=self.project_id))
        presynaptic_connectors = [k for k,v in self.connected_connectors.items() if len(v['presynaptic_to']) != 0]
        qs_tc = TreenodeConnector.objects.filter( project=self.project_id, connector__in=presynaptic_connectors, relation=relations['postsynaptic_to'] )
        res = {}
        for ele in qs_tc:
            if not ele.skeleton_id in res:
                res[ele.skeleton_id] = 0
            res[ele.skeleton_id] += 1
        return res

    def _fetch_upstream_skeletons(self):
        """ Returns a list of skeleton IDs that synapse onto this Skeleton.  """
        relations = dict((r.relation_name, r.id) for r in Relation.objects.filter(project=self.project_id))
        postsynaptic_connectors = [k for k,v in self.connected_connectors.items() if len(v['postsynaptic_to']) != 0]
        qs_tc = TreenodeConnector.objects.filter( project=self.project_id, connector__in=postsynaptic_connectors, relation=relations['presynaptic_to'] )
        res = {}
        for ele in qs_tc:
            if not ele.skeleton_id in res:
                res[ele.skeleton_id] = 0
            res[ele.skeleton_id] += 1
        return res

    def _create_graph(self):
        # retrieve all nodes of the skeleton
        treenode_qs = Treenode.objects.filter(
            skeleton_id=self.skeleton_id)
        # build the networkx graph from it
        graph = nx.DiGraph()
        for e in treenode_qs:
            graph.add_node( e.id )
            # TODO: add attributes
            graph.node[e.id] = {
                'user_id': e.user_id,
                'creation_time': e.creation_time,
                'edition_time': e.edition_time,
                'location': np.array([e.location.x, e.location.y, e.location.z], dtype=np.float32),
                'reviewer_id': e.reviewer_id,
                'review_time': e.review_time,
                'radius': e.radius,
                # TODO: labels!
            }
            if e.parent_id:
                graph.add_edge( e.parent_id, e.id, {'confidence': e.confidence} )
        return graph

    def cable_length(self):
        """ Compute the sum of the edge lengths which is the total cable length. """
        if self._edge_length_sum != 0.0:
            return self._edge_length_sum
        sum = 0.0
        node = self.graph.node
        for ID_from, ID_to in self.graph.edges(data=False):
            sum += np.linalg.norm(node[ID_to]['location'] - node[ID_from]['location'])
        self._edge_length_sum = sum
        return self._edge_length_sum
<commit_msg>Added tags to objects api<commit_after>from models import *

import numpy as np
import networkx as nx

class Neuron(object):

    def __init__(self, neuron_id, project_id):

        self.neuron = ClassInstance.objects.get(pk=neuron_id, project_id = project_id)
        self.neuron_id = neuron_id
        self.project_id = project_id

    def get_contained_skeletons(self):

        qs = ClassInstanceClassInstance.objects.filter(
            relation__relation_name='model_of',
            project=self.project_id,
            class_instance_b=self.neuron_id).select_related("class_instance_a__id")

        return [ele.class_instance_a.id for ele in qs]

class Skeleton(object):

    def __init__(self, skeleton_id, project_id = None):

        if project_id is None:
            self.skeleton = ClassInstance.objects.get(pk=skeleton_id)
            project_id = self.skeleton.project_id
        else:
            self.skeleton = ClassInstance.objects.get(pk=skeleton_id, project_id = project_id)

        self.skeleton_id = skeleton_id
        self.project_id = project_id
        self._edge_length_sum = 0.0

        self.graph = self._create_graph()
        self.connected_connectors = self._fetch_connected_connectors()
        self.downstream_skeletons = self._fetch_downstream_skeletons()
        self.upstream_skeletons = self._fetch_upstream_skeletons()

        self._compute_skeleton_edge_deltatime()

        qs = ClassInstanceClassInstance.objects.filter(
            relation__relation_name='model_of',
            project=self.project_id,
            class_instance_a=self.skeleton_id).select_related("class_instance_b")
        self.neuron = qs[0].class_instance_b

    def node_count(self):
        return self.graph.number_of_nodes()

    def input_count(self):
        """ Returns the number of anatomical synaptic inputs onto this Skeleton. """
        n = 0
        for k,v in self.upstream_skeletons.items():
            n += v
        return n

    def output_count(self):
        """ Returns the number of anatomical synaptic outputs onto other Skeleton instances. """
        n = 0
        for k,v in self.downstream_skeletons.items():
            n += v
        return n

    def _fetch_connected_connectors(self):
        """
        Example:
        { connector_id: {'presynaptic_to': [node_id1, node_id2],
                         'postsynaptic_to': [node_id3] }
        """

        qs_tc = TreenodeConnector.objects.filter(
            project=self.project_id,
            skeleton=self.skeleton.id
        ).select_related('connector')

        relations = dict((r.relation_name, r.id) for r in Relation.objects.filter(project=self.project_id))
        results = {}

        for tc in qs_tc:
            if not tc.connector_id in results:
                results[tc.connector_id] = {
                    'presynaptic_to': [],
                    'postsynaptic_to': [],
                    # TODO: labels, location etc.
                }

            if tc.relation_id == relations['presynaptic_to']:
                results[tc.connector_id]['presynaptic_to'].append( tc.treenode_id )
            elif tc.relation_id == relations['postsynaptic_to']:
                results[tc.connector_id]['postsynaptic_to'].append( tc.treenode_id )

        return results

    def _fetch_downstream_skeletons(self):
        """ Returns a list of skeleton IDs that this Skeleton synapses onto. """
        relations = dict((r.relation_name, r.id) for r in Relation.objects.filter(project=self.project_id))
        presynaptic_connectors = [k for k,v in self.connected_connectors.items() if len(v['presynaptic_to']) != 0]
        qs_tc = TreenodeConnector.objects.filter( project=self.project_id, connector__in=presynaptic_connectors, relation=relations['postsynaptic_to'] )
        res = {}
        for ele in qs_tc:
            if not ele.skeleton_id in res:
                res[ele.skeleton_id] = 0
            res[ele.skeleton_id] += 1
        return res

    def _fetch_upstream_skeletons(self):
        """ Returns a list of skeleton IDs that synapse onto this Skeleton.  """
        relations = dict((r.relation_name, r.id) for r in Relation.objects.filter(project=self.project_id))
        postsynaptic_connectors = [k for k,v in self.connected_connectors.items() if len(v['postsynaptic_to']) != 0]
        qs_tc = TreenodeConnector.objects.filter( project=self.project_id, connector__in=postsynaptic_connectors, relation=relations['presynaptic_to'] )
        res = {}
        for ele in qs_tc:
            if not ele.skeleton_id in res:
                res[ele.skeleton_id] = 0
            res[ele.skeleton_id] += 1
        return res

    def _create_graph(self):
        # retrieve all nodes of the skeleton
        treenode_qs = Treenode.objects.filter(
            skeleton_id=self.skeleton_id)
        # build the networkx graph from it
        graph = nx.DiGraph()
        for e in treenode_qs:
            graph.add_node( e.id )
            # TODO: add attributes
            graph.node[e.id] = {
                'user_id': e.user_id,
                'creation_time': e.creation_time,
                'edition_time': e.edition_time,
                'location': np.array([e.location.x, e.location.y, e.location.z], dtype=np.float32),
                'reviewer_id': e.reviewer_id,
                'review_time': e.review_time,
                'radius': e.radius,
                'tags': []
            }
            if e.parent_id:
                graph.add_edge( e.parent_id, e.id, {'confidence': e.confidence} )

        # add labels
        tci = TreenodeClassInstance.objects.filter(
            relation__relation_name='labeled_as',
            class_instance__class_column__class_name='label',
            treenode__in=graph.nodes(),
            project=self.project_id).select_related('class_instance__name')

        for t in tci:
            graph.node[t.treenode_id]['tags'].append( t.class_instance.name )

        return graph

    def cable_length(self):
        """ Compute the sum of the edge lengths which is the total cable length. """
        if self._edge_length_sum != 0.0:
            return self._edge_length_sum
        sum = 0.0
        node = self.graph.node
        for ID_from, ID_to in self.graph.edges(data=False):
            sum += np.linalg.norm(node[ID_to]['location'] - node[ID_from]['location'])
        self._edge_length_sum = sum
        return self._edge_length_sum

{'fraction_non_alphanumeric': 0.0980974552460464, 'fraction_numerical': 0.003211045998233925, 'mean_word_length': 2.288806758183738, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2962816', 'n_tokens_mistral': 3896, 'n_tokens_neox': 3651, 'n_words': 761}