<commit_before>from django.shortcuts import render
from ranker.models import Project, Item, Judge, Rating
from django.contrib.auth.models import User
#from sklearn import linear_model
# http://scikit-learn.org/stable/modules/linear_model.html#ordinary-least-squares
from django.http import HttpResponse, HttpResponseRedirect
from django.template import RequestContext, loader
from django.core.context_processors import csrf 
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required
from django.contrib.auth import authenticate, login
from django.contrib import messages
from ranker.forms import UserCreateForm, ProjectForm
from django.core.files.uploadedfile import SimpleUploadedFile
from scipy.optimize import fmin_bfgs, fmin_tnc, fmin_l_bfgs_b, check_grad
from django.db.models import Q
from math import exp, log
import numpy as np
import random
import csv

# regularization parameters
item_mean = 0.0
item_std = 10.0
judge_mean = 1.0
judge_std = 10.0 

def register(request):
    if request.method == 'POST':
        form = UserCreateForm(request.POST)     # create form object
        if form.is_valid():
            form.save()
            messages.info(request, "Thank you for registering. You are now logged in.")
            new_user = authenticate(username=request.POST['username'],
                                    password=request.POST['<PASSWORD>'])
            login(request, new_user)
            return HttpResponseRedirect(reverse('dashboard'))
    else:
        form = UserCreateForm()

    args = {}
    args.update(csrf(request))
    args['form'] = form
    return render(request, 'ranker/register.html', args)


def index(request):
    template = loader.get_template('ranker/index.html')
    context = RequestContext(request, {})
    return HttpResponse(template.render(context))

@login_required
def dashboard(request):
    projects = Project.objects.filter(user=request.user)
    template = loader.get_template('ranker/dashboard.html')
    context = RequestContext(request, {'projects': projects})
    return HttpResponse(template.render(context))

@login_required
def create_project(request):
    if request.method == 'POST':
        form = ProjectForm(request.POST, request.FILES)     # create form object

        if form.is_valid():
            instance = form.save(commit=False)
            instance.user = request.user
            instance.save()
            form.save_images()

            messages.success(request, "Project Created!")
            #return HttpResponseRedirect(reverse('dashboard'))
            return HttpResponseRedirect(reverse('view_project',
                                                kwargs={'project_id':
                                                        instance.id}))
    else:
        form = ProjectForm()

    args = {}
    args.update(csrf(request))
    args['form'] = form
    return render(request, 'ranker/create_project.html', args)

@login_required
def view_project(request, project_id):
    project = Project.objects.get(id=project_id)

    update_model(project_id)

    # todo need to ensure the judge parameters are right; particularly if they
    # vote on multiple projects
    judges = Judge.objects.filter(ratings__project=project).distinct()

    #update_model(project.id)
    #min_val = min([item.mean - item.conf for item in Item.objects.filter(project=project)])
    #max_val = max([item.mean + item.conf for item in Item.objects.filter(project=project)])
    template = loader.get_template('ranker/view_project.html')
    context = RequestContext(request, {'project': project,
                                       'judges': judges})
                                      # 'min_val': min_val,
                                      # 'max_val': max_val})
    return HttpResponse(template.render(context))

def expz(val):
    if val > 12:
        val = 12
    elif val < -12:
        val = -12
    return exp(val)

def ll_2p(x, *args):
    ids = {i:idx for idx, i in enumerate(args[1])}
    jids = {i:idx + len(ids) for idx, i in enumerate(args[0])}
    ratings = args[2]
    #ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids))

    ll = 0.0
    for r in ratings:
        left = x[ids[r.left.id]]
        right = x[ids[r.right.id]]
        d = x[jids[r.judge.id]]

        y = r.value
        z = d * (left - right)
        ez = expz(z)
        ll += y * z - log(1 + ez)

    # Normal prior on means
    item_reg = 0.0
    for i in ids:
        diff = x[ids[i]] - item_mean
        item_reg += diff * diff
    item_reg = (-1.0 / (2 * item_std * item_std)) * item_reg

    # Normal prior on discriminations
    judge_reg = 0.0
    for i in jids:
        diff = x[jids[i]] - judge_mean
        judge_reg += diff * diff
    judge_reg = (-1.0 / (2 * judge_std * judge_std)) * judge_reg

    return -1.0 * (ll + item_reg + judge_reg)
    #return -1.0 * (ll + item_reg)

def ll_2p_grad(x, *args):
    ids = {i:idx for idx, i in enumerate(args[1])}
    jids = {i:idx + len(ids) for idx, i in enumerate(args[0])}
    ratings = args[2]
    #ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids))

    grad = np.array([0.0 for v in x])
    for r in ratings:
        left = x[ids[r.left.id]]
        right = x[ids[r.right.id]]
        d = x[jids[r.judge.id]]
        y = r.value
        p = 1.0 / (1.0 + expz(-1 * d * (left-right)))

        g = y - p 
        grad[ids[r.left.id]] += d * g
        grad[ids[r.right.id]] += -1 * d * g
        grad[jids[r.judge.id]] += (left - right) * g

    # Normal prior on means
    item_reg = np.array([0.0 for v in x])
    for i in ids:
        item_reg[ids[i]] += (x[ids[i]] - item_mean)
    item_reg = (-1.0 / (item_std * item_std)) * item_reg

    # Gamma prior on discriminations
    judge_reg = np.array([0.0 for v in x])
    for i in jids:
        judge_reg[jids[i]] += (x[jids[i]] - judge_mean)
    judge_reg = (-1.0 / (judge_std * judge_std)) * judge_reg

    return -1 * (grad + item_reg + judge_reg)
        
def ll_1p(x, *args):
    ids = {i:idx for idx, i in enumerate(args[0])}
    ratings = args[1]

    ll = 0.0
    for r in ratings:
        left = x[ids[r.left.id]]
        right = x[ids[r.right.id]]
        y = r.value
        z = (left - right)
        ez = expz(z)
        ll += y * z - log(1 + ez)

    # Normal prior on means
    item_reg = 0.0
    for i in ids:
        diff = x[ids[i]] - item_mean
        item_reg += diff * diff
    item_reg = (-1.0 / (2 * item_std * item_std)) * item_reg

    return -1.0 * (ll + item_reg)

def ll_1p_grad(x, *args):
    ids = {i:idx for idx, i in enumerate(args[0])}
    ratings = args[1]

    grad = np.array([0.0 for v in x])
    for r in ratings:
        left = x[ids[r.left.id]]
        right = x[ids[r.right.id]]
        y = r.value
        p = 1.0 / (1.0 + expz(-1 * (left-right)))

        g = y - p 
        grad[ids[r.left.id]] += g
        grad[ids[r.right.id]] += -1 * g

    # Normal prior on means
    item_reg = np.array([0.0 for v in x])
    for i in ids:
        item_reg[ids[i]] += (x[ids[i]] - item_mean)
    item_reg = (-1.0 / (item_std * item_std)) * item_reg

    return -1 * (grad + item_reg)

#def invlogit(val):
#    return 1.0 / (1.0 + expz(val))

#def ll_plus_grad(x, *args):
#    ids = {i:idx for idx, i in enumerate(args)}
#    #print(ids)
#    ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids))
#
#    ll = 0.0
#    grad = [0.0 for v in x]
#    for r in ratings:
#        left = x[ids[r.left.id]]
#        right = x[ids[r.right.id]]
#        y = r.value
#        p = max(0.00001, min(0.99999, invlogit(left-right)))
#        ez = expz(left - right)
#
#        ll += (y * log(p) + (1 - y) * log(1-p))
#        g = ((y - p) * ez * p) / (1 - p)
#        grad[ids[r.left.id]] += g
#        grad[ids[r.right.id]] += -1 * g
#
#    return -1.0 * np.array(ll), -1.0 * np.array(grad)


# OLD, i think it returns the same parameters, but less precise. 
#def ll_1p(x, *args):
#    #print(args)
#    #args = args[0]
#    ids = {i:idx for idx, i in enumerate(args)}
#    #print(ids)
#    ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids))
#
#    ll = 0.0
#    for r in ratings:
#        left = x[ids[r.left.id]]
#        right = x[ids[r.right.id]]
#        y = r.value
#        p = max(0.00001, min(0.99999, invlogit(left-right)))
#        ll += (y * log(p) + (1 - y) * log(1-p))
#
#    return -1.0 * np.array(ll)
#
#def ll_1p_grad(x, *args):
#    #print(args)
#    #args = args[0]
#    ids = {i:idx for idx, i in enumerate(args)}
#    #print(ids)
#    ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids))
#
#    grad = [0.0 for v in x]
#    for r in ratings:
#        left = x[ids[r.left.id]]
#        right = x[ids[r.right.id]]
#        y = r.value
#        p = max(0.00001, min(0.99999, invlogit(left-right)))
#        ez = expz(-1(left - right))
#
#        g = ((y - p) * ez * p) / (1 - p)
#        grad[ids[r.left.id]] += g
#        grad[ids[r.right.id]] += -1 * g
#
#    return -1 * np.array(grad)

def update_model(project_id):
    project = Project.objects.get(id=project_id)
    items = Item.objects.filter(project=project).order_by('id').distinct()
    ids = [item.id for item in items]
    judges = Judge.objects.filter(ratings__project=project).order_by('id').distinct()
    jids = [judge.id for judge in judges]
    ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids)).distinct()
    #x0 = [item.mean for item in items] + [judge.discrimination for judge in judges]

    ######## 2PL #############
    x0 = [item.mean for item in items] + [judge.discrimination for judge in judges]
    #x0 = [0.0 for item in items] + [1.0 for judge in judges]
    # BFGS
    #result = fmin_bfgs(ll_2p, x0, fprime=ll_2p_grad, 
    #                   args=(tuple(jids), tuple(ids), ratings), disp=False)

    # Truncated Newton
    bounds = [('-inf','inf') for v in ids] + [(0.001,'inf') for v in jids]
    result = fmin_tnc(ll_2p, x0, 
                      #approx_grad=True,
                      fprime=ll_2p_grad, 
                      args=(tuple(jids), tuple(ids), ratings), bounds=bounds, disp=True)[0]
    ##########################

    ######## 1PL #############
    #x0 = [item.mean for item in items] 
    ## BFGS
    #result = fmin_bfgs(ll_1p, x0, fprime=ll_1p_grad, args=(tuple(ids), ratings),
    #                   disp=True)

    # Truncated Newton<commit_msg>re-initialize judge each time<commit_after>from django.shortcuts import render
from ranker.models import Project, Item, Judge, Rating
from django.contrib.auth.models import User
#from sklearn import linear_model
# http://scikit-learn.org/stable/modules/linear_model.html#ordinary-least-squares
from django.http import HttpResponse, HttpResponseRedirect
from django.template import RequestContext, loader
from django.core.context_processors import csrf 
from django.core.urlresolvers import reverse
from django.contrib.auth.decorators import login_required
from django.contrib.auth import authenticate, login
from django.contrib import messages
from ranker.forms import UserCreateForm, ProjectForm
from django.core.files.uploadedfile import SimpleUploadedFile
from scipy.optimize import fmin_bfgs, fmin_tnc, fmin_l_bfgs_b, check_grad
from django.db.models import Q
from math import exp, log
import numpy as np
import random
import csv

# regularization parameters
item_mean = 0.0
item_std = 10.0
judge_mean = 1.0
judge_std = 10.0 

def register(request):
    if request.method == 'POST':
        form = UserCreateForm(request.POST)     # create form object
        if form.is_valid():
            form.save()
            messages.info(request, "Thank you for registering. You are now logged in.")
            new_user = authenticate(username=request.POST['username'],
                                    password=request.POST['<PASSWORD>'])
            login(request, new_user)
            return HttpResponseRedirect(reverse('dashboard'))
    else:
        form = UserCreateForm()

    args = {}
    args.update(csrf(request))
    args['form'] = form
    return render(request, 'ranker/register.html', args)


def index(request):
    template = loader.get_template('ranker/index.html')
    context = RequestContext(request, {})
    return HttpResponse(template.render(context))

@login_required
def dashboard(request):
    projects = Project.objects.filter(user=request.user)
    template = loader.get_template('ranker/dashboard.html')
    context = RequestContext(request, {'projects': projects})
    return HttpResponse(template.render(context))

@login_required
def create_project(request):
    if request.method == 'POST':
        form = ProjectForm(request.POST, request.FILES)     # create form object

        if form.is_valid():
            instance = form.save(commit=False)
            instance.user = request.user
            instance.save()
            form.save_images()

            messages.success(request, "Project Created!")
            #return HttpResponseRedirect(reverse('dashboard'))
            return HttpResponseRedirect(reverse('view_project',
                                                kwargs={'project_id':
                                                        instance.id}))
    else:
        form = ProjectForm()

    args = {}
    args.update(csrf(request))
    args['form'] = form
    return render(request, 'ranker/create_project.html', args)

@login_required
def view_project(request, project_id):
    project = Project.objects.get(id=project_id)

    update_model(project_id)

    # todo need to ensure the judge parameters are right; particularly if they
    # vote on multiple projects
    judges = Judge.objects.filter(ratings__project=project).distinct()

    #update_model(project.id)
    #min_val = min([item.mean - item.conf for item in Item.objects.filter(project=project)])
    #max_val = max([item.mean + item.conf for item in Item.objects.filter(project=project)])
    template = loader.get_template('ranker/view_project.html')
    context = RequestContext(request, {'project': project,
                                       'judges': judges})
                                      # 'min_val': min_val,
                                      # 'max_val': max_val})
    return HttpResponse(template.render(context))

def expz(val):
    if val > 12:
        val = 12
    elif val < -12:
        val = -12
    return exp(val)

def ll_2p(x, *args):
    ids = {i:idx for idx, i in enumerate(args[1])}
    jids = {i:idx + len(ids) for idx, i in enumerate(args[0])}
    ratings = args[2]
    #ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids))

    ll = 0.0
    for r in ratings:
        left = x[ids[r.left.id]]
        right = x[ids[r.right.id]]
        d = x[jids[r.judge.id]]

        y = r.value
        z = d * (left - right)
        ez = expz(z)
        ll += y * z - log(1 + ez)

    # Normal prior on means
    item_reg = 0.0
    for i in ids:
        diff = x[ids[i]] - item_mean
        item_reg += diff * diff
    item_reg = (-1.0 / (2 * item_std * item_std)) * item_reg

    # Normal prior on discriminations
    judge_reg = 0.0
    for i in jids:
        diff = x[jids[i]] - judge_mean
        judge_reg += diff * diff
    judge_reg = (-1.0 / (2 * judge_std * judge_std)) * judge_reg

    return -1.0 * (ll + item_reg + judge_reg)
    #return -1.0 * (ll + item_reg)

def ll_2p_grad(x, *args):
    ids = {i:idx for idx, i in enumerate(args[1])}
    jids = {i:idx + len(ids) for idx, i in enumerate(args[0])}
    ratings = args[2]
    #ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids))

    grad = np.array([0.0 for v in x])
    for r in ratings:
        left = x[ids[r.left.id]]
        right = x[ids[r.right.id]]
        d = x[jids[r.judge.id]]
        y = r.value
        p = 1.0 / (1.0 + expz(-1 * d * (left-right)))

        g = y - p 
        grad[ids[r.left.id]] += d * g
        grad[ids[r.right.id]] += -1 * d * g
        grad[jids[r.judge.id]] += (left - right) * g

    # Normal prior on means
    item_reg = np.array([0.0 for v in x])
    for i in ids:
        item_reg[ids[i]] += (x[ids[i]] - item_mean)
    item_reg = (-1.0 / (item_std * item_std)) * item_reg

    # Gamma prior on discriminations
    judge_reg = np.array([0.0 for v in x])
    for i in jids:
        judge_reg[jids[i]] += (x[jids[i]] - judge_mean)
    judge_reg = (-1.0 / (judge_std * judge_std)) * judge_reg

    return -1 * (grad + item_reg + judge_reg)
        
def ll_1p(x, *args):
    ids = {i:idx for idx, i in enumerate(args[0])}
    ratings = args[1]

    ll = 0.0
    for r in ratings:
        left = x[ids[r.left.id]]
        right = x[ids[r.right.id]]
        y = r.value
        z = (left - right)
        ez = expz(z)
        ll += y * z - log(1 + ez)

    # Normal prior on means
    item_reg = 0.0
    for i in ids:
        diff = x[ids[i]] - item_mean
        item_reg += diff * diff
    item_reg = (-1.0 / (2 * item_std * item_std)) * item_reg

    return -1.0 * (ll + item_reg)

def ll_1p_grad(x, *args):
    ids = {i:idx for idx, i in enumerate(args[0])}
    ratings = args[1]

    grad = np.array([0.0 for v in x])
    for r in ratings:
        left = x[ids[r.left.id]]
        right = x[ids[r.right.id]]
        y = r.value
        p = 1.0 / (1.0 + expz(-1 * (left-right)))

        g = y - p 
        grad[ids[r.left.id]] += g
        grad[ids[r.right.id]] += -1 * g

    # Normal prior on means
    item_reg = np.array([0.0 for v in x])
    for i in ids:
        item_reg[ids[i]] += (x[ids[i]] - item_mean)
    item_reg = (-1.0 / (item_std * item_std)) * item_reg

    return -1 * (grad + item_reg)

#def invlogit(val):
#    return 1.0 / (1.0 + expz(val))

#def ll_plus_grad(x, *args):
#    ids = {i:idx for idx, i in enumerate(args)}
#    #print(ids)
#    ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids))
#
#    ll = 0.0
#    grad = [0.0 for v in x]
#    for r in ratings:
#        left = x[ids[r.left.id]]
#        right = x[ids[r.right.id]]
#        y = r.value
#        p = max(0.00001, min(0.99999, invlogit(left-right)))
#        ez = expz(left - right)
#
#        ll += (y * log(p) + (1 - y) * log(1-p))
#        g = ((y - p) * ez * p) / (1 - p)
#        grad[ids[r.left.id]] += g
#        grad[ids[r.right.id]] += -1 * g
#
#    return -1.0 * np.array(ll), -1.0 * np.array(grad)


# OLD, i think it returns the same parameters, but less precise. 
#def ll_1p(x, *args):
#    #print(args)
#    #args = args[0]
#    ids = {i:idx for idx, i in enumerate(args)}
#    #print(ids)
#    ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids))
#
#    ll = 0.0
#    for r in ratings:
#        left = x[ids[r.left.id]]
#        right = x[ids[r.right.id]]
#        y = r.value
#        p = max(0.00001, min(0.99999, invlogit(left-right)))
#        ll += (y * log(p) + (1 - y) * log(1-p))
#
#    return -1.0 * np.array(ll)
#
#def ll_1p_grad(x, *args):
#    #print(args)
#    #args = args[0]
#    ids = {i:idx for idx, i in enumerate(args)}
#    #print(ids)
#    ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids))
#
#    grad = [0.0 for v in x]
#    for r in ratings:
#        left = x[ids[r.left.id]]
#        right = x[ids[r.right.id]]
#        y = r.value
#        p = max(0.00001, min(0.99999, invlogit(left-right)))
#        ez = expz(-1(left - right))
#
#        g = ((y - p) * ez * p) / (1 - p)
#        grad[ids[r.left.id]] += g
#        grad[ids[r.right.id]] += -1 * g
#
#    return -1 * np.array(grad)

def update_model(project_id):
    project = Project.objects.get(id=project_id)
    items = Item.objects.filter(project=project).order_by('id').distinct()
    ids = [item.id for item in items]
    judges = Judge.objects.filter(ratings__project=project).order_by('id').distinct()
    jids = [judge.id for judge in judges]
    ratings = Rating.objects.filter(Q(left__in=ids)|Q(right__in=ids)).distinct()
    #x0 = [item.mean for item in items] + [judge.discrimination for judge in judges]

    ######## 2PL #############
    x0 = [item.mean for item in items] + [1.0 for judge in judges]
    #x0 = [0.0 for item in items] + [1.0 for judge in judges]
    # BFGS
    #result = fmin_bfgs(ll_2p, x0, fprime=ll_2p_grad, 
    #                   args=(tuple(jids), tuple(ids), ratings), disp=False)

    # Truncated Newton
    bounds = [('-inf','inf') for v in ids] + [(0.001,'inf') for v in jids]
    result = fmin_tnc(ll_2p, x0, 
                      fprime=ll_2p_grad, 
                      args=(tuple(jids), tuple(ids), ratings), bounds=bounds,
                      disp=False)[0]
    ##########################

    ######## 1PL #############
    #x0 = [item.mean for item in items] 
    ## BFGS
    #result = fmin_bfgs(ll_1p, x0, fprime=ll_1p_grad, args=(tuple(ids), ratings),
    #                   disp=True)

    # Truncated Newton
{'fraction_non_alphanumeric': 0.15006978870866824, 'fraction_numerical': 0.017519372382923424, 'mean_word_length': 2.359961190168176, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '865387', 'n_tokens_mistral': 7440, 'n_tokens_neox': 7201, 'n_words': 1757}