<commit_before>import sys
import json
from models import NeuronSearch, ClassInstance, Project, User, Treenode
from models import ClassInstanceClassInstance, Relation, Class
from collections import defaultdict
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.db import connection, transaction
from django.http import HttpResponse, HttpResponseRedirect
from django.core.urlresolvers import reverse

# Tip from: http://lincolnloop.com/blog/2008/may/10/getting-requestcontext-your-templates/
# Required because we need a RequestContext, not just a Context - the
# former looks at TEMPLATE_CONTEXT_PROCESSORS, while the latter doesn't.

def my_render_to_response(req, *args, **kwargs):
    kwargs['context_instance'] = RequestContext(req)
    return render_to_response(*args, **kwargs)

# Both index and visual_index take a request and kwargs and then
# return a list of neurons and a NeuronSearch form:

def get_form_and_neurons(request, project_id, kwargs):
    # If we've been passed parameters in a REST-style GET request,
    # create a form from them.  Otherwise, if it's a POST request,
    # create the form from the POST parameters.  Otherwise, it's a
    # plain request, so create the default search form.
    rest_keys = ('search','cell_body_location','order_by')
    if any((x in kwargs) for x in rest_keys):
        kw_search = kwargs.get('search',None) or ""
        kw_cell_body_choice = kwargs.get('cell_body_location',None) or "-1"
        kw_order_by = kwargs.get('order_by',None) or 'name'
        search_form = NeuronSearch({'search': kw_search,
                                    'cell_body_location': kw_cell_body_choice,
                                    'order_by': kw_order_by })
    elif request.method == 'POST':
        search_form = NeuronSearch(request.POST)
    else:
        search_form = NeuronSearch({'search': '',
                                    'cell_body_location': -1,
                                    'order_by': 'name'})

    if search_form.is_valid():
        search = search_form.cleaned_data['search']
        cell_body_location = int(search_form.cleaned_data['cell_body_location'])
        order_by = search_form.cleaned_data['order_by']
    else:
        search = ''
        cell_body_location = -1
        order_by = 'name'

    all_neurons = Neuron.objects.filter(name__icontains=search)
    if cell_body_location >= 0:
        all_neurons = all_neurons.filter(cell_body=cell_body_location)
    all_neurons = order_neuron_queryset(all_neurons,order_by)
    return ( all_neurons, search_form )

def index(request, **kwargs):
    all_neurons, search_form = get_form_and_neurons(request,
                                                    kwargs['project_id'],
                                                    kwargs)
    return my_render_to_response(request,
                                 'vncbrowser/index.html',
                                 {'all_neurons_list': all_neurons,
                                  'search_form': search_form})

def visual_index(request, **kwargs):

    all_neurons, search_form = get_form_and_neurons( request, kwargs )

    # From: http://docs.djangoproject.com/en/1.0/topics/pagination/
    paginator = Paginator(all_neurons, 20)
    if 'page' in kwargs:
        page = kwargs['page'] or 1
    else:
        try:
            page = int(request.GET.get('page', '1'))
        except ValueError:
            page = 1

    # If page request (9999) is out of range, deliver last page of results.
    try:
        neurons = paginator.page(page)
    except (EmptyPage, InvalidPage):
        neurons = paginator.page(paginator.num_pages)

    return my_render_to_response(request,
                                 'vncbrowser/visual_index.html',
                                 {'sorted_neurons': neurons.object_list.all(),
                                  'sorted_neurons_page' : neurons,
                                  'search_form': search_form })

def group_neurons_descending_count(neurons):
    id_to_neurons = defaultdict(set)
    for neuron in neurons:
        id_to_neurons[neuron.id].add(neuron)
    reverse_sorted = sorted(id_to_neurons.items(),
                            key=lambda x: -len(id_to_neurons[x[0]]))
    result = []
    for neuron_id, neurons in reverse_sorted:
        count = len(neurons)
        neuron = list(neurons)[0]
        neuron.count = count
        result.append(neuron)
    return result

def view(request, project_id=None, neuron_id=None, neuron_name=None):
    p = get_object_or_404(Project, pk=project_id)
    if neuron_id:
        n = get_object_or_404(ClassInstance, pk=neuron_id, project=project_id)
    else:
        n = get_object_or_404(ClassInstance, name=neuron_name, project=project_id)

    lines = ClassInstance.objects.filter(
        project=p,
        class_instances_a__class_instance_b=n,
        class_instances_a__relation__relation_name='expresses_in').all()

    outgoing = group_neurons_descending_count(
        ClassInstance.all_neurons_downstream(n))
    incoming = group_neurons_descending_count(
        ClassInstance.all_neurons_upstream(n))

    return my_render_to_response(request,
                                 'vncbrowser/view.html',
                                 {'neuron': n,
                                  'lines': lines,
                                  'project_id': project_id,
                                  'incoming': incoming,
                                  'outgoing': outgoing} )
<commit_msg>Add a reminder to do stricter checking of classes<commit_after>import sys
import json
from models import NeuronSearch, ClassInstance, Project, User, Treenode
from models import ClassInstanceClassInstance, Relation, Class
from collections import defaultdict
from django.shortcuts import render_to_response, get_object_or_404
from django.template import RequestContext
from django.db import connection, transaction
from django.http import HttpResponse, HttpResponseRedirect
from django.core.urlresolvers import reverse

# Tip from: http://lincolnloop.com/blog/2008/may/10/getting-requestcontext-your-templates/
# Required because we need a RequestContext, not just a Context - the
# former looks at TEMPLATE_CONTEXT_PROCESSORS, while the latter doesn't.

def my_render_to_response(req, *args, **kwargs):
    kwargs['context_instance'] = RequestContext(req)
    return render_to_response(*args, **kwargs)

# Both index and visual_index take a request and kwargs and then
# return a list of neurons and a NeuronSearch form:

def get_form_and_neurons(request, project_id, kwargs):
    # If we've been passed parameters in a REST-style GET request,
    # create a form from them.  Otherwise, if it's a POST request,
    # create the form from the POST parameters.  Otherwise, it's a
    # plain request, so create the default search form.
    rest_keys = ('search','cell_body_location','order_by')
    if any((x in kwargs) for x in rest_keys):
        kw_search = kwargs.get('search',None) or ""
        kw_cell_body_choice = kwargs.get('cell_body_location',None) or "-1"
        kw_order_by = kwargs.get('order_by',None) or 'name'
        search_form = NeuronSearch({'search': kw_search,
                                    'cell_body_location': kw_cell_body_choice,
                                    'order_by': kw_order_by })
    elif request.method == 'POST':
        search_form = NeuronSearch(request.POST)
    else:
        search_form = NeuronSearch({'search': '',
                                    'cell_body_location': -1,
                                    'order_by': 'name'})

    if search_form.is_valid():
        search = search_form.cleaned_data['search']
        cell_body_location = int(search_form.cleaned_data['cell_body_location'])
        order_by = search_form.cleaned_data['order_by']
    else:
        search = ''
        cell_body_location = -1
        order_by = 'name'

    all_neurons = Neuron.objects.filter(name__icontains=search)
    if cell_body_location >= 0:
        all_neurons = all_neurons.filter(cell_body=cell_body_location)
    all_neurons = order_neuron_queryset(all_neurons,order_by)
    return ( all_neurons, search_form )

def index(request, **kwargs):
    all_neurons, search_form = get_form_and_neurons(request,
                                                    kwargs['project_id'],
                                                    kwargs)
    return my_render_to_response(request,
                                 'vncbrowser/index.html',
                                 {'all_neurons_list': all_neurons,
                                  'search_form': search_form})

def visual_index(request, **kwargs):

    all_neurons, search_form = get_form_and_neurons( request, kwargs )

    # From: http://docs.djangoproject.com/en/1.0/topics/pagination/
    paginator = Paginator(all_neurons, 20)
    if 'page' in kwargs:
        page = kwargs['page'] or 1
    else:
        try:
            page = int(request.GET.get('page', '1'))
        except ValueError:
            page = 1

    # If page request (9999) is out of range, deliver last page of results.
    try:
        neurons = paginator.page(page)
    except (EmptyPage, InvalidPage):
        neurons = paginator.page(paginator.num_pages)

    return my_render_to_response(request,
                                 'vncbrowser/visual_index.html',
                                 {'sorted_neurons': neurons.object_list.all(),
                                  'sorted_neurons_page' : neurons,
                                  'search_form': search_form })

def group_neurons_descending_count(neurons):
    id_to_neurons = defaultdict(set)
    for neuron in neurons:
        id_to_neurons[neuron.id].add(neuron)
    reverse_sorted = sorted(id_to_neurons.items(),
                            key=lambda x: -len(id_to_neurons[x[0]]))
    result = []
    for neuron_id, neurons in reverse_sorted:
        count = len(neurons)
        neuron = list(neurons)[0]
        neuron.count = count
        result.append(neuron)
    return result

def view(request, project_id=None, neuron_id=None, neuron_name=None):
    p = get_object_or_404(Project, pk=project_id)
    # FIXME: add the class name as well
    if neuron_id:
        n = get_object_or_404(ClassInstance, pk=neuron_id, project=project_id)
    else:
        n = get_object_or_404(ClassInstance, name=neuron_name, project=project_id)

    lines = ClassInstance.objects.filter(
        project=p,
        class_instances_a__class_instance_b=n,
        class_instances_a__relation__relation_name='expresses_in').all()

    outgoing = group_neurons_descending_count(
        ClassInstance.all_neurons_downstream(n))
    incoming = group_neurons_descending_count(
        ClassInstance.all_neurons_upstream(n))

    return my_render_to_response(request,
                                 'vncbrowser/view.html',
                                 {'neuron': n,
                                  'lines': lines,
                                  'project_id': project_id,
                                  'incoming': incoming,
                                  'outgoing': outgoing} )

{'fraction_non_alphanumeric': 0.08826156869751653, 'fraction_numerical': 0.006253350008933357, 'mean_word_length': 2.385243423042032, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4008554', 'n_tokens_mistral': 3316, 'n_tokens_neox': 3193, 'n_words': 719}