<commit_before># -*- coding: utf-8 -*-
import datetime
import textwrap
import bisect
import re
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.core.urlresolvers import reverse
from django.db import IntegrityError
from django.db.models import Q, Max, Count
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import redirect
from django.template import RequestContext
from django.template import Template
from django.template.loader import get_template
from reversion.models import Revision, Version
# from reversion import revision
from ielex.settings import LIMIT_TO
from ielex.forms import *
from ielex.lexicon.models import *
# from ielex.citations.models import *
from ielex.extensional_semantics.views import *
from ielex.shortcuts import render_template
from ielex.utilities import next_alias, confirm_required, anchored, oneline

############ CHANGED ##############################

from collections import defaultdict
from django.views.decorators.csrf import csrf_protect
from django_tables2 import RequestConfig
from ielex.tables import *
from werkzeug.datastructures import MultiDict

##################################################

# Refactoring: 
# - rename the functions which render to response with the format
# view_TEMPLATE_NAME(request, ...). Put subsiduary functions under their main
# caller.

# -- Database input, output and maintenance functions ------------------------

def view_changes(request, username=None, revision_id=None, object_id=None):
    """Recent changes"""
    boring_models = [LanguageListOrder, LanguageList, MeaningList]
    boring_model_ids = [ContentType.objects.get_for_model(m).id for m in
            boring_models]
    def interesting_versions(self):
        return self.version_set.exclude(content_type_id__in=boring_model_ids)
    Revision.add_to_class("interesting_versions", interesting_versions)

    if not username:
        recent_changes = Revision.objects.all().order_by("-id")
    else:
        recent_changes = Revision.objects.filter(
                user__username=username).order_by("-id")
    paginator = Paginator(recent_changes, 50) # was 200

    try: # Make sure page request is an int. If not, deliver first page.
        page = int(request.GET.get('page', '1'))
    except ValueError:
        page = 1

    try: # If page request is out of range, deliver last page of results.
        changes = paginator.page(page)
    except (EmptyPage, InvalidPage):
        changes = paginator.page(paginator.num_pages)

    contributors = sorted([(User.objects.get(id=user_id),
            Revision.objects.filter(user=user_id).count())
            for user_id in Revision.objects.values_list("user",
            flat=True).distinct() if user_id is not None],
            lambda x, y: -cmp(x[1], y[1])) # reverse sort by second element in tuple
            # TODO user_id should never be None

    return render_template(request, "view_changes.html",
            {"changes":changes,
            "contributors":contributors})

@login_required
def revert_version(request, revision_id):
    """Roll back the object saved in a Version to the previous Version"""
    # TODO
    # - redirect this to somewhere more useful
    # - get the rollback revision and add a useful comment
    referer = request.META.get("HTTP_REFERER", "/")
    revision_obj = Revision.objects.get(pk=revision_id)
    revision_obj.revert() # revert all associated objects too
    msg = "Rolled back revision %s" % (revision_obj.id)
    messages.add_message(request, messages.INFO, msg)
    return HttpResponseRedirect(referer)

# TODO
def view_object_history(request, version_id):
    version = Version.objects.get(pk=version_id)
    obj = version.content_type.get_object_for_this_type(id=version.object_id)
    # versions = Version.objects.get_for_object(
    #         latest.content_type.get_object_for_this_type(
    #         id=latest.object_id)).filter(id__lt=version_id).reverse()
    fields = [field.name for field in obj._meta.fields]
    versions = [[v.field_dict[f] for f in fields]+[v.id] for v in
            Version.objects.get_for_object(obj).order_by("revision__date_created")]
    return render_template(request, "view_object_history.html",
            {"object":obj,
            "versions":versions,
            "fields":fields})


# -- General purpose queries and functions -----------------------------------

def get_canonical_meaning(meaning):
    """Identify meaning from id number or partial name"""
    try:
        if meaning.isdigit():
            meaning = Meaning.objects.get(id=meaning)
        else:
            meaning = Meaning.objects.get(gloss=meaning)
    except Meaning.DoesNotExist:
        raise Http404("Meaning '%s' does not exist" % meaning)
    return meaning

def get_canonical_language(language, request=None):
    """Identify language from id number or partial name"""
    if not language:
        raise Language.DoesNotExist
    if language.isdigit():
        language = Language.objects.get(id=language)
    else:
        try:
            language = Language.objects.get(ascii_name=language)
        except Language.DoesNotExist:
            try:
                language = Language.objects.get(
                    ascii_name__istartswith=language)
            except Language.MultipleObjectsReturned:
                if request:
                    messages.add_message(request, messages.INFO,
                            ("There are multiple languages matching"\
                            " '%s' in the database") % language)
                raise Http404
            except Language.DoesNotExist:
                if request:
                    messages.add_message(request, messages.INFO,
                            ("There is no language named or starting"\
                            " with '%s' in the database") % language)
                raise Http404
    return language

def get_current_language_list_name(request):
    """Get the name of the current language list from session. This is
    only be be used by navigation functions (e.g.
    get_previous_and_next_languages) which don't take part in the RESTful url
    scheme"""
    return request.session.get("language_list_name", LanguageList.DEFAULT)

def get_prev_and_next_languages(request, current_language, language_list=None):
    # XXX language_list argument is not currently dispatched to this function
    # TODO this needs to be fixed (solution: get it from session variable)
    if language_list:
        language_list = LanguageList.objects.get(name=language_list)
    else:
        language_list = LanguageList.objects.get(name=LanguageList.DEFAULT)

    ids = list(language_list.languages.values_list(
            "id", flat=True).order_by("languagelistorder"))
    try:
        current_idx = ids.index(current_language.id)
    except ValueError:
        current_idx = 0
    prev_language = Language.objects.get(id=ids[current_idx-1])
    try:
        next_language = Language.objects.get(id=ids[current_idx+1])
    except IndexError:
        next_language = Language.objects.get(id=ids[0])
    return (prev_language, next_language)

def get_prev_and_next_meanings(request, current_meaning):
    # We'll let this one use the session variable (kind of cheating...)
    meaning_list = request.session.get("current_wordlist_name", MeaningList.DEFAULT)
    meaning_list = MeaningList.objects.get(name=meaning_list)
    meanings = list(meaning_list.meanings.all().order_by("meaninglistorder"))

    ids = [m.id for m in meanings]
    current_idx = ids.index(current_meaning.id)
    prev_meaning = meanings[current_idx-1]
    # prev_meaning = Meaning.objects.get(id=ids[current_idx-1])
    try:
        next_meaning = meanings[current_idx+1]
        # next_meaning = Meaning.objects.get(id=ids[current_idx+1])
    except IndexError:
        next_meaning = meanings[0]
        # next_meaning = Meaning.objects.get(id=ids[0])
    return (prev_meaning, next_meaning)

def get_prev_and_next_lexemes(request, current_lexeme):
    """Get the previous and next lexeme from the same language, ordered
    by meaning and then alphabetically by form"""
    current_meaning = current_lexeme.meaning
    lexemes = list(Lexeme.objects.filter(language=current_lexeme.language).order_by(
            "meaning", "phon_form", "source_form", "id"))
    ids = [l.id for l in lexemes]
    current_idx = ids.index(current_lexeme.id)
    prev_lexeme = lexemes[current_idx-1]
    try:
        next_lexeme = lexemes[current_idx+1]
    except IndexError:
        next_lexeme = lexemes[0]
    return (prev_lexeme, next_lexeme)

def update_object_from_form(model_object, form):
    """Update an object with data from a form."""
    # XXX This is only neccessary when not using a model form: otherwise
    # form.save() does all this automatically
    # TODO Refactor this function away
    assert set(form.cleaned_data).issubset(set(model_object.__dict__))
    model_object.__dict__.update(form.cleaned_data)
    model_object.save()
    return

# -- /language(s)/ ----------------------------------------------------------

def get_canonical_language_list(language_list=None, request=None):
    """Returns a LanguageList object"""
    try:
        if language_list is None:
            language_list = LanguageList.objects.get(name=LanguageList.DEFAULT)
        elif language_list.isdigit():
            language_list = LanguageList.objects.get(id=language_list)
        else:
            language_list = LanguageList.objects.get(name=language_list)
    except LanguageList.DoesNotExist:
        if request:
            messages.add_message(request, messages.INFO,
                    ("There is no language list matching"\
                    " '%s' in the database") % language_list)
        raise Http404
    return language_list

# def get_language_list_form(request):
#     language_list_name = get_current_language_list_name(request)
#     if request.method == 'POST':
#         form = ChooseLanguageListForm(request.POST)
#         if form.is_valid():
#             current_list = form.cleaned_data["language_list"]
#             language_list_name = current_list.name
#             msg = "Language list selection changed to '%s'" %\
#                     language_list_name
#             messages.add_message(request, messages.INFO, msg)
#     else:
#         form = ChooseLanguageListForm()
#     form.fields["language_list"].initial = LanguageList.objects.get(
#             name=language_list_name).id
#     return form


############ CHANGED ##############################

@csrf_protect
def view_language_list(request, language_list=None):
    current_list = get_canonical_language_list(language_list, request)
    request.session["current_language_list_name"] = current_list.name
    languages = current_list.languages.all().order_by("languagelistorder")
    languages = languages.annotate(lexeme_count=Count("lexeme"))

    if request.method == 'POST':
        form = ChooseLanguageListForm(request.POST)
        if form.is_valid():
            current_list = form.cleaned_data["language_list"]
            request.session["current_language_list_name"] = current_list.name
            msg = u"Language list selection changed to ‘%s’" %\
                    current_list.name
            messages.add_message(request, messages.INFO, msg)
            return HttpResponseRedirect(reverse("view-language-list",
                    args=[current_list.name]))
    else:
        form = ChooseLanguageListForm()
    form.fields["language_list"].initial = current_list.id

    return render_template(request, "language_list.html",
            {"languages":languages,
            "language_list_form":form,
            "current_list":current_list})

@csrf_protect
def view_language_listALT(request, language_list=None):
    current_list = get_canonical_language_list(language_list, request)
    request.session["current_language_list_name"] = current_list.name
    languages = current_list.languages.all().order_by("languagelistorder")
    languages = languages.annotate(lexeme_count=Count("lexeme"))

    def process_postrequest_form(multidict):
        res = defaultdict(list)
        for key in multidict.keys():
            if not(key in ['langlist_form', 'csrfmiddlewaretoken']):
                outer_key = ''.join(key.split('-')[0:2])
                inner_key = key.split('-')[-1]
                res[outer_key].append((inner_key, multidict.getlist(key)[0]))
        return res

    def is_unchanged(lang, vdict):
        <commit_msg>Put editable function for languagelist into production.<commit_after># -*- coding: utf-8 -*-
import datetime
import textwrap
import bisect
import re
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.core.paginator import Paginator, InvalidPage, EmptyPage
from django.core.urlresolvers import reverse
from django.db import IntegrityError
from django.db.models import Q, Max, Count
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import redirect
from django.template import RequestContext
from django.template import Template
from django.template.loader import get_template
from reversion.models import Revision, Version
# from reversion import revision
from ielex.settings import LIMIT_TO
from ielex.forms import *
from ielex.lexicon.models import *
# from ielex.citations.models import *
from ielex.extensional_semantics.views import *
from ielex.shortcuts import render_template
from ielex.utilities import next_alias, confirm_required, anchored, oneline

############ CHANGED ##############################

from collections import defaultdict
from django.views.decorators.csrf import csrf_protect
from django_tables2 import RequestConfig
from ielex.tables import *
from werkzeug.datastructures import MultiDict

##################################################

# Refactoring: 
# - rename the functions which render to response with the format
# view_TEMPLATE_NAME(request, ...). Put subsiduary functions under their main
# caller.

# -- Database input, output and maintenance functions ------------------------

def view_changes(request, username=None, revision_id=None, object_id=None):
    """Recent changes"""
    boring_models = [LanguageListOrder, LanguageList, MeaningList]
    boring_model_ids = [ContentType.objects.get_for_model(m).id for m in
            boring_models]
    def interesting_versions(self):
        return self.version_set.exclude(content_type_id__in=boring_model_ids)
    Revision.add_to_class("interesting_versions", interesting_versions)

    if not username:
        recent_changes = Revision.objects.all().order_by("-id")
    else:
        recent_changes = Revision.objects.filter(
                user__username=username).order_by("-id")
    paginator = Paginator(recent_changes, 50) # was 200

    try: # Make sure page request is an int. If not, deliver first page.
        page = int(request.GET.get('page', '1'))
    except ValueError:
        page = 1

    try: # If page request is out of range, deliver last page of results.
        changes = paginator.page(page)
    except (EmptyPage, InvalidPage):
        changes = paginator.page(paginator.num_pages)

    contributors = sorted([(User.objects.get(id=user_id),
            Revision.objects.filter(user=user_id).count())
            for user_id in Revision.objects.values_list("user",
            flat=True).distinct() if user_id is not None],
            lambda x, y: -cmp(x[1], y[1])) # reverse sort by second element in tuple
            # TODO user_id should never be None

    return render_template(request, "view_changes.html",
            {"changes":changes,
            "contributors":contributors})

@login_required
def revert_version(request, revision_id):
    """Roll back the object saved in a Version to the previous Version"""
    # TODO
    # - redirect this to somewhere more useful
    # - get the rollback revision and add a useful comment
    referer = request.META.get("HTTP_REFERER", "/")
    revision_obj = Revision.objects.get(pk=revision_id)
    revision_obj.revert() # revert all associated objects too
    msg = "Rolled back revision %s" % (revision_obj.id)
    messages.add_message(request, messages.INFO, msg)
    return HttpResponseRedirect(referer)

# TODO
def view_object_history(request, version_id):
    version = Version.objects.get(pk=version_id)
    obj = version.content_type.get_object_for_this_type(id=version.object_id)
    # versions = Version.objects.get_for_object(
    #         latest.content_type.get_object_for_this_type(
    #         id=latest.object_id)).filter(id__lt=version_id).reverse()
    fields = [field.name for field in obj._meta.fields]
    versions = [[v.field_dict[f] for f in fields]+[v.id] for v in
            Version.objects.get_for_object(obj).order_by("revision__date_created")]
    return render_template(request, "view_object_history.html",
            {"object":obj,
            "versions":versions,
            "fields":fields})


# -- General purpose queries and functions -----------------------------------

def get_canonical_meaning(meaning):
    """Identify meaning from id number or partial name"""
    try:
        if meaning.isdigit():
            meaning = Meaning.objects.get(id=meaning)
        else:
            meaning = Meaning.objects.get(gloss=meaning)
    except Meaning.DoesNotExist:
        raise Http404("Meaning '%s' does not exist" % meaning)
    return meaning

def get_canonical_language(language, request=None):
    """Identify language from id number or partial name"""
    if not language:
        raise Language.DoesNotExist
    if language.isdigit():
        language = Language.objects.get(id=language)
    else:
        try:
            language = Language.objects.get(ascii_name=language)
        except Language.DoesNotExist:
            try:
                language = Language.objects.get(
                    ascii_name__istartswith=language)
            except Language.MultipleObjectsReturned:
                if request:
                    messages.add_message(request, messages.INFO,
                            ("There are multiple languages matching"\
                            " '%s' in the database") % language)
                raise Http404
            except Language.DoesNotExist:
                if request:
                    messages.add_message(request, messages.INFO,
                            ("There is no language named or starting"\
                            " with '%s' in the database") % language)
                raise Http404
    return language

def get_current_language_list_name(request):
    """Get the name of the current language list from session. This is
    only be be used by navigation functions (e.g.
    get_previous_and_next_languages) which don't take part in the RESTful url
    scheme"""
    return request.session.get("language_list_name", LanguageList.DEFAULT)

def get_prev_and_next_languages(request, current_language, language_list=None):
    # XXX language_list argument is not currently dispatched to this function
    # TODO this needs to be fixed (solution: get it from session variable)
    if language_list:
        language_list = LanguageList.objects.get(name=language_list)
    else:
        language_list = LanguageList.objects.get(name=LanguageList.DEFAULT)

    ids = list(language_list.languages.values_list(
            "id", flat=True).order_by("languagelistorder"))
    try:
        current_idx = ids.index(current_language.id)
    except ValueError:
        current_idx = 0
    prev_language = Language.objects.get(id=ids[current_idx-1])
    try:
        next_language = Language.objects.get(id=ids[current_idx+1])
    except IndexError:
        next_language = Language.objects.get(id=ids[0])
    return (prev_language, next_language)

def get_prev_and_next_meanings(request, current_meaning):
    # We'll let this one use the session variable (kind of cheating...)
    meaning_list = request.session.get("current_wordlist_name", MeaningList.DEFAULT)
    meaning_list = MeaningList.objects.get(name=meaning_list)
    meanings = list(meaning_list.meanings.all().order_by("meaninglistorder"))

    ids = [m.id for m in meanings]
    current_idx = ids.index(current_meaning.id)
    prev_meaning = meanings[current_idx-1]
    # prev_meaning = Meaning.objects.get(id=ids[current_idx-1])
    try:
        next_meaning = meanings[current_idx+1]
        # next_meaning = Meaning.objects.get(id=ids[current_idx+1])
    except IndexError:
        next_meaning = meanings[0]
        # next_meaning = Meaning.objects.get(id=ids[0])
    return (prev_meaning, next_meaning)

def get_prev_and_next_lexemes(request, current_lexeme):
    """Get the previous and next lexeme from the same language, ordered
    by meaning and then alphabetically by form"""
    current_meaning = current_lexeme.meaning
    lexemes = list(Lexeme.objects.filter(language=current_lexeme.language).order_by(
            "meaning", "phon_form", "source_form", "id"))
    ids = [l.id for l in lexemes]
    current_idx = ids.index(current_lexeme.id)
    prev_lexeme = lexemes[current_idx-1]
    try:
        next_lexeme = lexemes[current_idx+1]
    except IndexError:
        next_lexeme = lexemes[0]
    return (prev_lexeme, next_lexeme)

def update_object_from_form(model_object, form):
    """Update an object with data from a form."""
    # XXX This is only neccessary when not using a model form: otherwise
    # form.save() does all this automatically
    # TODO Refactor this function away
    assert set(form.cleaned_data).issubset(set(model_object.__dict__))
    model_object.__dict__.update(form.cleaned_data)
    model_object.save()
    return

# -- /language(s)/ ----------------------------------------------------------

def get_canonical_language_list(language_list=None, request=None):
    """Returns a LanguageList object"""
    try:
        if language_list is None:
            language_list = LanguageList.objects.get(name=LanguageList.DEFAULT)
        elif language_list.isdigit():
            language_list = LanguageList.objects.get(id=language_list)
        else:
            language_list = LanguageList.objects.get(name=language_list)
    except LanguageList.DoesNotExist:
        if request:
            messages.add_message(request, messages.INFO,
                    ("There is no language list matching"\
                    " '%s' in the database") % language_list)
        raise Http404
    return language_list

# def get_language_list_form(request):
#     language_list_name = get_current_language_list_name(request)
#     if request.method == 'POST':
#         form = ChooseLanguageListForm(request.POST)
#         if form.is_valid():
#             current_list = form.cleaned_data["language_list"]
#             language_list_name = current_list.name
#             msg = "Language list selection changed to '%s'" %\
#                     language_list_name
#             messages.add_message(request, messages.INFO, msg)
#     else:
#         form = ChooseLanguageListForm()
#     form.fields["language_list"].initial = LanguageList.objects.get(
#             name=language_list_name).id
#     return form


############ CHANGED ##############################

@csrf_protect
def view_language_listALT(request, language_list=None):
    current_list = get_canonical_language_list(language_list, request)
    request.session["current_language_list_name"] = current_list.name
    languages = current_list.languages.all().order_by("languagelistorder")
    languages = languages.annotate(lexeme_count=Count("lexeme"))

    if request.method == 'POST':
        form = ChooseLanguageListForm(request.POST)
        if form.is_valid():
            current_list = form.cleaned_data["language_list"]
            request.session["current_language_list_name"] = current_list.name
            msg = u"Language list selection changed to ‘%s’" %\
                    current_list.name
            messages.add_message(request, messages.INFO, msg)
            return HttpResponseRedirect(reverse("view-language-list",
                    args=[current_list.name]))
    else:
        form = ChooseLanguageListForm()
    form.fields["language_list"].initial = current_list.id

    return render_template(request, "language_list.html",
            {"languages":languages,
            "language_list_form":form,
            "current_list":current_list})

@csrf_protect
def view_language_list(request, language_list=None):
    current_list = get_canonical_language_list(language_list, request)
    request.session["current_language_list_name"] = current_list.name
    languages = current_list.languages.all().order_by("languagelistorder")
    languages = languages.annotate(lexeme_count=Count("lexeme"))

    def process_postrequest_form(multidict):
        res = defaultdict(list)
        for key in multidict.keys():
            if not(key in ['langlist_form', 'csrfmiddlewaretoken']):
                outer_key = ''.join(key.split('-')[0:2])
                inner_key = key.split('-')[-1]
                res[outer_key].append((inner_key, multidict.getlist(key)[0]))
        return res

    def is_unchanged(lang, vdict):
        
{'fraction_non_alphanumeric': 0.11106322964820184, 'fraction_numerical': 0.0033691138447073574, 'mean_word_length': 3.1426484907497567, 'pattern_counts': {'":': 16, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1301092', 'n_tokens_mistral': 7284, 'n_tokens_neox': 7054, 'n_words': 1694}