<commit_before>import json
from django.db import connection
from django.http import HttpResponse
from catmaid.control.authentication import *
from catmaid.models import Relation
import networkx as nx
from networkx.algorithms import weakly_connected_component_subgraphs
from collections import defaultdict
from itertools import chain, ifilter, imap, izip, count
from functools import partial
from synapseclustering import tree_max_density
from numpy import subtract
from numpy.linalg import norm
from tree_util import edge_count_to_root, simplify, find_root, reroot, partition, spanning_tree, cable_length
from math import sqrt
import sys
from operator import getitem

def basic_graph(project_id, skeleton_ids):
    cursor = connection.cursor()

    cursor.execute('''
    SELECT relation_name, id
    FROM relation
    WHERE project_id = %s
      AND (relation_name = 'presynaptic_to' OR relation_name = 'postsynaptic_to')
    ''' % int(project_id))
    relations = dict(cursor.fetchall())
    preID, postID = relations['presynaptic_to'], relations['postsynaptic_to']

    cursor.execute('''
    SELECT tc.connector_id, tc.relation_id, tc.skeleton_id
    FROM treenode_connector tc
    WHERE tc.project_id = %s
      AND tc.skeleton_id IN (%s)
    ''' % (int(project_id), ",".join(str(int(skid)) for skid in skeleton_ids)))

    # stores entire query set in memory, linking pre and post
    connectors = defaultdict(partial(defaultdict, list))
    for row in cursor.fetchall():
        connectors[row[0]][row[1]].append(row[2])

    # Safe to placeholder, half-complete connectors:
    # only adds the edge if both pre and post exist
    edges = defaultdict(partial(defaultdict, int))
    for c in connectors.itervalues():
        for pre in c[preID]: # should be one or none
            for post in c[postID]:
                edges[pre][post] += 1

    return {'edges': tuple((pre, post, count) for pre, edge in edges.iteritems() for post, count in edge.iteritems())}

    '''
    return {'edges': [{'source': pre,
                       'target': post,
                       'weight': count} for pre, edge in edges.iteritems() for post, count in edge.iteritems()]}
    '''

    """ Can't get the variable to be set with all the skeleton IDs
    cursor.execute('''
    WITH skeletons as (VALUES (%s) UNION ALL)
    SELECT tc1.skeleton_id, tc2.skeleton_id, count(*)
    FROM treenode_connector tc1,
         treenode_connector tc2
    WHERE tc1.project_id = %s
      AND tc1.project_id = tc2.project_id
      AND tc1.connector_id = tc2.connector_id
      AND tc1.relation_id = %s
      AND tc2.relation_id = %s
      AND tc1.skeleton_id IN skeletons
      AND tc2.skeleton_id IN skeletons
    ''' % (",".join(str(int(skid)) for skid in skeleton_ids), int(project_id), preID, postID))
    """


def confidence_split_graph(project_id, skeleton_ids, confidence_threshold):
    """ Assumes 0 < confidence_threshold <= 5. """
    cursor = connection.cursor()
    skids = ",".join(str(int(skid)) for skid in skeleton_ids)

    cursor.execute('''
    SELECT relation_name, id
    FROM relation
    WHERE project_id = %s
      AND (relation_name = 'presynaptic_to' OR relation_name = 'postsynaptic_to')
    ''' % int(project_id))
    relations = dict(cursor.fetchall())
    preID, postID = relations['presynaptic_to'], relations['postsynaptic_to']

    # Fetch synapses of all skeletons
    cursor.execute('''
    SELECT skeleton_id, treenode_id, connector_id, relation_id
    FROM treenode_connector
    WHERE project_id = %s
      AND skeleton_id IN (%s)
    ''' % (int(project_id), skids))

    stc = defaultdict(list)
    for row in cursor.fetchall():
        stc[row[0]].append(row[1:]) # skeleton_id vs (treenode_id, connector_id, relation_id)

    # Fetch all treenodes of all skeletons
    cursor.execute('''
    SELECT skeleton_id, id, parent_id, confidence
    FROM treenode
    WHERE project_id = %s
      AND skeleton_id IN (%s)
    ORDER BY skeleton_id
    ''' % (project_id, skids))

    # Dictionary of connector_id vs relation_id vs list of sub-skeleton ID
    connectors = defaultdict(partial(defaultdict, list))

    # All nodes of the graph
    nodeIDs = []

    # Read out into memory only one skeleton at a time
    current_skid = None
    tree = None
    for row in cursor.fetchall():
        if row[0] == current_skid:
            # Build the tree, breaking it at the low-confidence edges
            if row[2] and row[3] >= confidence_threshold:
                    tree.add_edge(row[2], row[1])
            continue

        if tree:
            nodeIDs.extend(split_by_confidence(current_skid, tree, stc[current_skid], connectors))

        # Start the next tree
        current_skid = row[0]
        tree = nx.DiGraph()
        if row[2] and row[3] > confidence_threshold:
            tree.add_edge(row[2], row[1])

    if tree:
        nodeIDs.extend(split_by_confidence(current_skid, tree, stc[current_skid], connectors))

    # Create the edges of the graph from the connectors, which was populated as a side effect of 'split_by_confidence'
    edges = defaultdict(partial(defaultdict, int)) # pre vs post vs count
    for c in connectors.itervalues():
        for pre in c[preID]:
            for post in c[postID]:
                edges[pre][post] += 1

    return {'nodes': nodeIDs,
            'edges': [(pre, post, count) for pre, edge in edges.iteritems() for post, count in edge.iteritems()]}


def dual_split_graph(project_id, skeleton_ids, confidence_threshold, bandwidth, expand):
    """ Assumes bandwidth > 0 and some skeleton_id in expand. """
    cursor = connection.cursor()
    skeleton_ids = set(skeleton_ids)
    expand = set(expand)

    # assumes all skeleton_id in expand are also present in skeleton_ids

    skids = ",".join(str(int(skid)) for skid in skeleton_ids)

    cursor.execute('''
    SELECT relation_name, id<commit_msg>Graph module: raise errors if there are parameters missing<commit_after>import json
from django.db import connection
from django.http import HttpResponse
from catmaid.control.authentication import *
from catmaid.models import Relation
import networkx as nx
from networkx.algorithms import weakly_connected_component_subgraphs
from collections import defaultdict
from itertools import chain, ifilter, imap, izip, count
from functools import partial
from synapseclustering import tree_max_density
from numpy import subtract
from numpy.linalg import norm
from tree_util import edge_count_to_root, simplify, find_root, reroot, partition, spanning_tree, cable_length
from math import sqrt
import sys
from operator import getitem

def basic_graph(project_id, skeleton_ids):
    if not skeleton_ids:
        raise ValueError("No skeleton IDs provided")

    cursor = connection.cursor()

    cursor.execute('''
    SELECT relation_name, id
    FROM relation
    WHERE project_id = %s
      AND (relation_name = 'presynaptic_to' OR relation_name = 'postsynaptic_to')
    ''' % int(project_id))
    relations = dict(cursor.fetchall())
    preID, postID = relations['presynaptic_to'], relations['postsynaptic_to']

    cursor.execute('''
    SELECT tc.connector_id, tc.relation_id, tc.skeleton_id
    FROM treenode_connector tc
    WHERE tc.project_id = %s
      AND tc.skeleton_id IN (%s)
    ''' % (int(project_id), ",".join(str(int(skid)) for skid in skeleton_ids)))

    # stores entire query set in memory, linking pre and post
    connectors = defaultdict(partial(defaultdict, list))
    for row in cursor.fetchall():
        connectors[row[0]][row[1]].append(row[2])

    # Safe to placeholder, half-complete connectors:
    # only adds the edge if both pre and post exist
    edges = defaultdict(partial(defaultdict, int))
    for c in connectors.itervalues():
        for pre in c[preID]: # should be one or none
            for post in c[postID]:
                edges[pre][post] += 1

    return {'edges': tuple((pre, post, count) for pre, edge in edges.iteritems() for post, count in edge.iteritems())}

    '''
    return {'edges': [{'source': pre,
                       'target': post,
                       'weight': count} for pre, edge in edges.iteritems() for post, count in edge.iteritems()]}
    '''

    """ Can't get the variable to be set with all the skeleton IDs
    cursor.execute('''
    WITH skeletons as (VALUES (%s) UNION ALL)
    SELECT tc1.skeleton_id, tc2.skeleton_id, count(*)
    FROM treenode_connector tc1,
         treenode_connector tc2
    WHERE tc1.project_id = %s
      AND tc1.project_id = tc2.project_id
      AND tc1.connector_id = tc2.connector_id
      AND tc1.relation_id = %s
      AND tc2.relation_id = %s
      AND tc1.skeleton_id IN skeletons
      AND tc2.skeleton_id IN skeletons
    ''' % (",".join(str(int(skid)) for skid in skeleton_ids), int(project_id), preID, postID))
    """


def confidence_split_graph(project_id, skeleton_ids, confidence_threshold):
    """ Assumes 0 < confidence_threshold <= 5. """
    if not skeleton_ids:
        raise ValueError("No skeleton IDs provided")

    cursor = connection.cursor()
    skids = ",".join(str(int(skid)) for skid in skeleton_ids)

    cursor.execute('''
    SELECT relation_name, id
    FROM relation
    WHERE project_id = %s
      AND (relation_name = 'presynaptic_to' OR relation_name = 'postsynaptic_to')
    ''' % int(project_id))
    relations = dict(cursor.fetchall())
    preID, postID = relations['presynaptic_to'], relations['postsynaptic_to']

    # Fetch synapses of all skeletons
    cursor.execute('''
    SELECT skeleton_id, treenode_id, connector_id, relation_id
    FROM treenode_connector
    WHERE project_id = %s
      AND skeleton_id IN (%s)
    ''' % (int(project_id), skids))

    stc = defaultdict(list)
    for row in cursor.fetchall():
        stc[row[0]].append(row[1:]) # skeleton_id vs (treenode_id, connector_id, relation_id)

    # Fetch all treenodes of all skeletons
    cursor.execute('''
    SELECT skeleton_id, id, parent_id, confidence
    FROM treenode
    WHERE project_id = %s
      AND skeleton_id IN (%s)
    ORDER BY skeleton_id
    ''' % (project_id, skids))

    # Dictionary of connector_id vs relation_id vs list of sub-skeleton ID
    connectors = defaultdict(partial(defaultdict, list))

    # All nodes of the graph
    nodeIDs = []

    # Read out into memory only one skeleton at a time
    current_skid = None
    tree = None
    for row in cursor.fetchall():
        if row[0] == current_skid:
            # Build the tree, breaking it at the low-confidence edges
            if row[2] and row[3] >= confidence_threshold:
                    tree.add_edge(row[2], row[1])
            continue

        if tree:
            nodeIDs.extend(split_by_confidence(current_skid, tree, stc[current_skid], connectors))

        # Start the next tree
        current_skid = row[0]
        tree = nx.DiGraph()
        if row[2] and row[3] > confidence_threshold:
            tree.add_edge(row[2], row[1])

    if tree:
        nodeIDs.extend(split_by_confidence(current_skid, tree, stc[current_skid], connectors))

    # Create the edges of the graph from the connectors, which was populated as a side effect of 'split_by_confidence'
    edges = defaultdict(partial(defaultdict, int)) # pre vs post vs count
    for c in connectors.itervalues():
        for pre in c[preID]:
            for post in c[postID]:
                edges[pre][post] += 1

    return {'nodes': nodeIDs,
            'edges': [(pre, post, count) for pre, edge in edges.iteritems() for post, count in edge.iteritems()]}


def dual_split_graph(project_id, skeleton_ids, confidence_threshold, bandwidth, expand):
    """ Assumes bandwidth > 0 and some skeleton_id in expand. """
    cursor = connection.cursor()
    skeleton_ids = set(skeleton_ids)
    expand = set(expand)

    if not skeleton_ids:
        raise ValueError("No skeleton IDs provided")

    # assumes all skeleton_id in expand are also present in skeleton_ids

    skids = ",".join(str(int(skid)) for skid in skeleton_ids)

    cursor.execute('''
    SELECT relation_name, id
{'fraction_non_alphanumeric': 0.09988366295496094, 'fraction_numerical': 0.005484460694698354, 'mean_word_length': 3.2346938775510203, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3128755', 'n_tokens_mistral': 3794, 'n_tokens_neox': 3606, 'n_words': 1127}