<commit_before># coding=UTF-8
import json, math, os, urllib

from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.core.mail import send_mail
from django.db.models import Count
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, render_to_response
from django.template import RequestContext
from django.utils.html import strip_tags
from django.utils.translation import ugettext as _
from public_project.forms import *
from public_project.models import *
from public_project.search import get_query, search_for_documents


def get_research_request(request):
    if 'research_request_id' in request.GET:
        try:
            research_request = ResearchRequest.objects.get(pk=request.GET['research_request_id'])
            return [research_request,]
        except ResearchRequest.DoesNotExist:
            pass
    return None


def get_user_comment(request):
    if 'comment_id' in request.GET:
        try:
            comment = Comment.objects.get(pk=request.GET['comment_id'])
            return [comment,]
        except Comment.DoesNotExist:
            pass
    return None
    

def validate_research_request_form(request):
    
    if request.method == 'POST' and 'research_request_form' in request.POST:
        if not request.user.has_perm('public_project.can_add_research_request'):
            return "FAILED"
        rr = ResearchRequest()
        rr.nr = request.POST['nr']
        rr.title = request.POST['title']
        rr.description = request.POST['description']
        rr.save()
            
        co1_content_type = ContentType.objects.get(app_label="public_project", model=request.POST['co1_content_type'])
        co1 = co1_content_type.get_object_for_this_type(id=request.POST['co1_id'])
        rr_rel = ResearchRequestRelation()
        rr_rel.research_request = rr
        rr_rel.content_object = co1
        if(request.POST['co1_content_type'] == 'document'):
            rr_rel.page = request.POST['co1_page']
        rr_rel.save()
            
        counter_more_cos = int(request.POST['counter_more_cos'])
            
        for num_id in range(2, counter_more_cos + 2):
            if 'co' + str(num_id) + '_id' in request.POST:
                co_content_type = ContentType.objects.get(app_label="public_project", model=request.POST['co' + str(num_id) + '_content_type'])
                co = co_content_type.get_object_for_this_type(id=request.POST['co' + str(num_id) + '_id'])
                rr_rel = ResearchRequestRelation()
                rr_rel.research_request = rr
                rr_rel.content_object = co
                if(request.POST['co' + str(num_id) + '_content_type'] == 'document'):
                    rr_rel.page = request.POST['co' + str(num_id) + '_page']
                rr_rel.save()        
        return "SENT"
    
    return ""


def validate_comment_form(request):
    
    if request.method == 'POST' and 'comment_form' in request.POST:
        print request.POST
        form = CommentForm(request.POST)
        if form.is_valid():
            c = Comment()
            c.username = form.cleaned_data['username']
            c.email = form.cleaned_data['email']
            c.comment = form.cleaned_data['comment']
            c.feedback_allowed = form.cleaned_data['feedback_allowed']
            c.activation_hash = os.urandom(16).encode('hex')
            c.save()
            
            co1_content_type = ContentType.objects.get(app_label="public_project", model=form.cleaned_data['co1_content_type'])
            co1 = co1_content_type.get_object_for_this_type(id=form.cleaned_data['co1_id'])
            cr = CommentRelation()
            cr.comment = c
            cr.content_object = co1
            if(form.cleaned_data['co1_content_type'] == 'document'):
                cr.page = form.cleaned_data['co1_page']
            cr.save()
            
            counter_more_cos = int(request.POST['counter_more_cos'])
            
            for num_id in range(2, counter_more_cos + 2):
                if 'co' + str(num_id) + '_id' in request.POST:
                    co_content_type = ContentType.objects.get(app_label="public_project", model=request.POST['co' + str(num_id) + '_content_type'])
                    co = co_content_type.get_object_for_this_type(id=request.POST['co' + str(num_id) + '_id'])
                    cr = CommentRelation()
                    cr.comment = c
                    cr.content_object = co
                    if(request.POST['co' + str(num_id) + '_content_type'] == 'document'):
                        cr.page = request.POST['co' + str(num_id) + '_page']
                    cr.save()
            
            email_users = User.objects.filter(userprofile__receive_new_comment_emails=True)
            
            #try:
            for i in range(0,1):
                for user in email_users:
                    sep = "-----------------------------------------------------------\n"
                    subject = _("NEW_COMMENT_EMAIL_SUBJECT") + ': ' + unicode(c)
                    
                    msg  = _("NEW_COMMENT_EMAIL_MESSAGE") + "\n"
                    msg += 'http://%s%s' % (Site.objects.get_current().domain, c.get_absolute_url()) + "\n\n"
                    msg += sep
                    msg += _("Name") + ": " + unicode(c.username) + "\n"
                    msg += _("E-Mail") + ": " + unicode(c.email) + "\n"
                    if c.feedback_allowed:
                        fa_str = _("yes")
                    else:
                        fa_str = _("no")
                    msg += _("Questions via mail allowed") + ": " + fa_str + "\n"
                    msg += _("Comment") + ":\n"
                    msg += c.comment + "\n" + sep
                    
                    msg += _("Comment on") + ":" + "\n"
                    for cr in c.commentrelation_set.all():
                        msg += unicode(cr.content_object) + "\n"
                        msg += 'http://%s%s' % (Site.objects.get_current().domain, cr.content_object.get_absolute_url()) + "\n"
                    
                    msg += "\n"
                    
                    if user.has_perm('public_project.change_comment') and user.email:
                        msg += _("NEW_COMMENT_EMAIL_MESSAGE_ACTIVATION") + "\n"
                        msg += 'http://%s/%s?activation_hash=%s&user=%s' \
                            % (Site.objects.get_current().domain, _("activate_comment_url"), c.activation_hash, urllib.quote_plus(unicode(user))) + "\n"
                    
                    send_mail(subject, msg, settings.EMAIL_FROM, [user.email], fail_silently=False)
            #except AttributeError:
            #    pass
            
            return "SENT"
        else:
            print form.errors
            return "FAILED"
    return ""


def merge_with_search_tag_docs(document_list, object):
    for tag in object.search_tags.all():
        search_docs = search_for_documents(tag.name)
        for doc in search_docs:
            if doc in document_list:
                index = document_list.index(doc)
                if not hasattr(document_list[index], 'search_tags'):
                    document_list[index].search_tags = []
                document_list[index].search_tags.append(tag.name)
            else:
                document_list.append(doc)
    document_list.sort(key=lambda x:x.title)
    return document_list


def index(request):
    research_request_list = ResearchRequest.objects.all()
    comment_list = Comment.objects.filter(published=True)
    
    if Event.objects.count() > 0:
        latest_event = Event.objects.all()[0]
    else:
        latest_event = None
    if Document.objects.count() > 0:
        latest_document = Document.objects.all()[0]
    else:
        latest_document = None 
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='home')[0],
        'current_project_goal_group': ProjectGoalGroup.objects.get_current(),
        'project_part_list': ProjectPart.objects.all(),
        'latest_event': latest_event,
        'latest_document': latest_document,
        'activity_list': ActivityLog.objects.all()[0:5],
        'research_request_list': research_request_list[0:1],
        'num_total_research_requests': len(research_request_list),
        'comment_list': comment_list[0:1],
        'num_total_comments': len(comment_list),
    })
    return render_to_response('index.html', context)


def project_parts(request):
    site_category = SiteCategory.objects.get_or_create(category='project_parts')[0]
    
    main_project_parts = ProjectPart.objects.annotate(count=Count('main_project_parts')).filter(count=0)
    middle = int(math.ceil(float(len(main_project_parts))/float(2)))
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': site_category,
        'main_project_part_list_left': main_project_parts[0:middle],
        'main_project_part_list_right': main_project_parts[middle:],
    })
    return render_to_response('project_parts.html', context)


def project_part(request, project_part_id):
    project_part = get_object_or_404(ProjectPart, pk=project_part_id)
    
    document_list = project_part.get_documents()
    document_list = merge_with_search_tag_docs(list(document_list), project_part)
    
    comment_form_status = validate_comment_form(request)
    
    content_type = ContentType.objects.get(app_label="public_project", model="projectpart")
    comment_list = Comment.objects.filter(commentrelation__content_type=content_type).filter(commentrelation__object_id=project_part.id).filter(published=True).distinct()
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='project_parts')[0],
        'user_comment': get_user_comment(request),
        'project_part': project_part,
        'question_list': project_part.get_questions(),
        'event_list': project_part.get_events(),
        'document_list': document_list,
        'comment_form_status': comment_form_status,
        'comment_list': comment_list[0:3],
        'num_total_comments': len(comment_list),
    })
    return render_to_response('project_part.html', context)


def goals(request):
    
    all_main_project_parts = ProjectPart.objects.annotate(count=Count('main_project_parts')).filter(count=0)
    main_project_parts = []
    for main_pp in all_main_project_parts:
        if main_pp.projectgoalgroup_set.count() > 0:
            main_project_parts.append(main_pp)
    middle = int(math.ceil(float(len(main_project_parts))/float(2)))
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='goals')[0],
        'common_goal_group_list': ProjectGoalGroup.objects.filter(project_part=None).order_by('event'),
        'main_project_part_list_left': main_project_parts[0:middle],
        'main_project_part_list_right': main_project_parts[middle:],
    })
    return render_to_response('goals.html', context)


def questions(request):
    main_project_parts = ProjectPart.objects.annotate(count=Count('main_project_parts')).filter(count=0)
    middle = int(math.ceil(float(len(main_project_parts))/float(2)))
    
    research_request_list = ResearchRequest.objects.all()
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='questions')[0],
        'main_project_part_list_left': main_project_parts[0:middle],
        'main_project_part_list_right': main_project_parts[middle:],
        'research_request_list': research_request_list[0:3],
        'num_total_research_requests': len(research_request_list),
    })
    return render_to_response('questions.html', context)


def question(request, question_id):
    question = get_object_or_404(Question, pk=question_id)
    
    research_request_form_status = validate_research_request_form(request)
    content_type = ContentType.objects.get(app_label="public_project", model="question")
    research_request_list = ResearchRequest.objects.filter(researchrequestrelation__content_type=content_type).filter(researchrequestrelation__object_id=question.id).distinct()
    
    comment_form_status = validate_comment_form(request)
    content_type = ContentType.objects.get(app_label="public_project", model="question")
    comment_list = Comment.objects.filter(commentrelation__content_type=content_type).filter(commentrelation__object_id=question.id).filter(published=True).distinct()
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='questions')[0],
        'user_comment': get_user_comment(request),
        'research_request': get_research_request(request),
        'question': question,
        'research_request_form_status': research_request_form_status,
        'research_request_list': research_request_list[0:3],
        'num_total_research_requests': len(research_request_list),
        'comment_form_status': comment_form_status,
        'comment_list': comment_list[0:3],
        'num_total_comments': len(comment_list),
    })
    return render_to_response('question.html', context)


def participants(request):
    site_category = SiteCategory.objects.get_or_create(category='participants')[0]
    groups = Participant.objects.annotate(count=Count('belongs_to')).filter(count=0)
    middle = int(math.ceil(float(len(groups))/float(2)))
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': site_category,
        'group_list_left': groups[0:middle],
        'group_list_right': groups[middle:],
    })
    return render_to_response('participants.html', context)


def participant(request, participant_id):
    participant = get_object_or_404(Participant, pk=participant_id)
    
    document_list = participant.get_documents()
    document_list = merge_with_search_tag_docs(list(document_list), participant)
    
    comment_form_status = validate_comment_form(request)
    content_type = ContentType.objects.get(app_label="public_project", model="participant")
    comment_list = Comment.objects.filter(commentrelation__content_type=content_type).filter(commentrelation__object_id=participant.id).filter(published=True).distinct()
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='participants')[0],
        'user_comment': get_user_comment(request),
        'participant': participant,
        'question_list': participant.get_questions(),
        'event_list': participant.get_events(),
        'document_list': document_list,
        'comment_form_status': comment_form_status,
        'comment_list': comment_list[0:3],
        'num_total_comments': len(comment_list),
    })
    return render_to_response('participant.html', context)


def events(request):
    site_category = SiteCategory.objects.get_or_create(category='events')[0]
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='events')[0],
        'project_goal_group_list': ProjectGoalGroup.objects.all().order_by('event'),
        'chronology_list': Event.objects.all(),
    })
    return render_to_response('events.html', context)


def event(request, event_id):
    event = get_object_or_404(Event, pk=event_id)
    
    document_list = list(event.related_documents.order_by("title"))
    document_list = merge_with_search_tag_docs(document_list, event)
    
    comment_form_status = validate_comment_form(request)
    content_type = ContentType.objects.get(app_label="public_project", model="event")
    comment_list = Comment.objects.filter(commentrelation__content_type=content_type).filter(commentrelation__object_id=event.id).filter(published=True).distinct()
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='events')[0],
        'user_comment': get_user_comment(request),
        'event': event,
        'document_list': document_list,
        'comment_form_status': comment_form_status,
        'comment_list': comment_list[0:3],
        'num_total_comments': len(comment_list),
    })
    return render_to_response('event.html', context)


def documents(request):
    site_category = SiteCategory.objects.get_or_create(category='documents')[0]
    
    main_project_parts = ProjectPart.objects.annotate(count=Count('main_project_parts')).filter(count=0)
    middle = int(math.ceil(float(len(main_project_parts))/float(2)))
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': site_category,
        'main_project_part_list_left': main_project_parts[0:middle],
        'main_project_part_list_right': main_project_parts[middle:],
    })
    return render_to_response('documents.html', context)


def xhr_universal_search(request):
    if request.method == 'GET' and 'query' in request.GET:
        query_string = request.GET['query']
        
        entry_query = get_query(query_string, ['name', 'main_project_part__name',])
        pp_list = list(ProjectPart.objects.select_related().filter(entry_query)[0:10])
        
        entry_query = get_query(query_string, ['title',])
        q_list = list(Question.objects.select_related().filter(entry_query)[0:10])
        
        entry_query = get_query(query_string, ['name',])
        p_list = list(Participant.objects.select_related().filter(entry_query)[0:10])
        
        entry_query = get_query(query_string, ['title',])
        e_list = list(Event.objects.select_related().filter(entry_query)[0:10])
        
        entry_query = get_query(query_string, ['title',])
        d_list = list(Document.objects.select_related().filter(entry_query)[0:10])
        
        object_list = pp_list + q_list + p_list + e_list + d_list
        
        res = {
            'values': {},
            'options': [],
        }
        
        for object in object_list:
            content_type = object.__class__.__name__.lower()
            id = object.id
            
            if not (content_type == request.GET['ommit_content_type'] and str(id) == request.GET['ommit_id']):
                option = '<i class="' + object.get_icon_class() + '"></i> ' + unicode(object)
                res['values'][option] = {
                    'content_type': content_type,
                    'id': id,
                }
                res['options'].append(option)
        
        mimetype = 'application/javascript'
        return HttpResponse(json.dumps(res), mimetype)


def xhr_document_tags(request):
    if request.method == 'POST':
        
        colors = {
            'projectpart': '#0d9434',
            'participant': '#3e3ec7',
            'event': '#c91a1a',
        }
        if request.POST['document_id']:
            document = get_object_or_404(Document, pk=request.POST['document_id'])
            if request.POST['content_type']:
                cache_entries = SearchTagCacheEntry.objects.filter(document=document).filter(tag__content_type__model=request.POST['content_type'])
            else:
                cache_entries = SearchTagCacheEntry.objects.filter(document=document)
            cache_entries = cache_entries[0:16]
            size_start = 14
            size_span = 10
        else:
            cache_entries = []
            if request.POST['content_type']:
                search_tags = SearchTag.objects.filter(content_type__model=request.POST['content_type']).annotate(num_cache_entries=Count('searchtagcacheentry')).order_by('-num_cache_entries')[0:22]
            else:
                search_tags = SearchTag.objects.annotate(num_cache_entries=Count('searchtagcacheentry')).order_by('-num_cache_entries')[0:22]
            for st in search_tags:
                if(st.searchtagcacheentry_set.count() > 0):
                    ce = st.searchtagcacheentry_set.all()[0]
                    ce.num_results = st.searchtagcacheentry_set.count()
                    cache_entries.append(ce)
            size_start = 16
            size_span = 12
        
        if len(cache_entries) > 0:
            first = cache_entries[0]
            max_num_results = first.num_results
        
        words = []
        for ce in cache_entries:
            text = ce.tag.name
            size = size_start + int(round((float(ce.num_results)/ float(max_num_results)) * size_span))
            color = colors[ce.tag.content_type.model]
            
            word = { "text": text, "size":size, "color": color, }
            words.append(word)
        
        data = json.dumps(words)
        
        mimetype = 'application/javascript'
        return HttpResponse(data, mimetype)


def document(request, document_id):
    document = get_object_or_404(Document, pk=document_id)
    comment_form_status = validate_comment_form(request)
    
    query_string = ''
    found_pages = []
    if ('q' in request.GET) and request.GET['q'].strip():
        query_string = request.GET['q']
        entry_query = get_query(query_string, ['document__title', 'content',])
        found_pages = Page.objects.select_related().filter(document=document).filter(entry_query).order_by('number')<commit_msg>Fixed bug (hopefully) with double tag cloud tags<commit_after># coding=UTF-8
import json, math, os, urllib

from django.conf import settings
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.core.mail import send_mail
from django.db.models import Count
from django.http import HttpResponse
from django.shortcuts import get_object_or_404, render_to_response
from django.template import RequestContext
from django.utils.html import strip_tags
from django.utils.translation import ugettext as _
from public_project.forms import *
from public_project.models import *
from public_project.search import get_query, search_for_documents


def get_research_request(request):
    if 'research_request_id' in request.GET:
        try:
            research_request = ResearchRequest.objects.get(pk=request.GET['research_request_id'])
            return [research_request,]
        except ResearchRequest.DoesNotExist:
            pass
    return None


def get_user_comment(request):
    if 'comment_id' in request.GET:
        try:
            comment = Comment.objects.get(pk=request.GET['comment_id'])
            return [comment,]
        except Comment.DoesNotExist:
            pass
    return None
    

def validate_research_request_form(request):
    
    if request.method == 'POST' and 'research_request_form' in request.POST:
        if not request.user.has_perm('public_project.can_add_research_request'):
            return "FAILED"
        rr = ResearchRequest()
        rr.nr = request.POST['nr']
        rr.title = request.POST['title']
        rr.description = request.POST['description']
        rr.save()
            
        co1_content_type = ContentType.objects.get(app_label="public_project", model=request.POST['co1_content_type'])
        co1 = co1_content_type.get_object_for_this_type(id=request.POST['co1_id'])
        rr_rel = ResearchRequestRelation()
        rr_rel.research_request = rr
        rr_rel.content_object = co1
        if(request.POST['co1_content_type'] == 'document'):
            rr_rel.page = request.POST['co1_page']
        rr_rel.save()
            
        counter_more_cos = int(request.POST['counter_more_cos'])
            
        for num_id in range(2, counter_more_cos + 2):
            if 'co' + str(num_id) + '_id' in request.POST:
                co_content_type = ContentType.objects.get(app_label="public_project", model=request.POST['co' + str(num_id) + '_content_type'])
                co = co_content_type.get_object_for_this_type(id=request.POST['co' + str(num_id) + '_id'])
                rr_rel = ResearchRequestRelation()
                rr_rel.research_request = rr
                rr_rel.content_object = co
                if(request.POST['co' + str(num_id) + '_content_type'] == 'document'):
                    rr_rel.page = request.POST['co' + str(num_id) + '_page']
                rr_rel.save()        
        return "SENT"
    
    return ""


def validate_comment_form(request):
    
    if request.method == 'POST' and 'comment_form' in request.POST:
        print request.POST
        form = CommentForm(request.POST)
        if form.is_valid():
            c = Comment()
            c.username = form.cleaned_data['username']
            c.email = form.cleaned_data['email']
            c.comment = form.cleaned_data['comment']
            c.feedback_allowed = form.cleaned_data['feedback_allowed']
            c.activation_hash = os.urandom(16).encode('hex')
            c.save()
            
            co1_content_type = ContentType.objects.get(app_label="public_project", model=form.cleaned_data['co1_content_type'])
            co1 = co1_content_type.get_object_for_this_type(id=form.cleaned_data['co1_id'])
            cr = CommentRelation()
            cr.comment = c
            cr.content_object = co1
            if(form.cleaned_data['co1_content_type'] == 'document'):
                cr.page = form.cleaned_data['co1_page']
            cr.save()
            
            counter_more_cos = int(request.POST['counter_more_cos'])
            
            for num_id in range(2, counter_more_cos + 2):
                if 'co' + str(num_id) + '_id' in request.POST:
                    co_content_type = ContentType.objects.get(app_label="public_project", model=request.POST['co' + str(num_id) + '_content_type'])
                    co = co_content_type.get_object_for_this_type(id=request.POST['co' + str(num_id) + '_id'])
                    cr = CommentRelation()
                    cr.comment = c
                    cr.content_object = co
                    if(request.POST['co' + str(num_id) + '_content_type'] == 'document'):
                        cr.page = request.POST['co' + str(num_id) + '_page']
                    cr.save()
            
            email_users = User.objects.filter(userprofile__receive_new_comment_emails=True)
            
            #try:
            for i in range(0,1):
                for user in email_users:
                    sep = "-----------------------------------------------------------\n"
                    subject = _("NEW_COMMENT_EMAIL_SUBJECT") + ': ' + unicode(c)
                    
                    msg  = _("NEW_COMMENT_EMAIL_MESSAGE") + "\n"
                    msg += 'http://%s%s' % (Site.objects.get_current().domain, c.get_absolute_url()) + "\n\n"
                    msg += sep
                    msg += _("Name") + ": " + unicode(c.username) + "\n"
                    msg += _("E-Mail") + ": " + unicode(c.email) + "\n"
                    if c.feedback_allowed:
                        fa_str = _("yes")
                    else:
                        fa_str = _("no")
                    msg += _("Questions via mail allowed") + ": " + fa_str + "\n"
                    msg += _("Comment") + ":\n"
                    msg += c.comment + "\n" + sep
                    
                    msg += _("Comment on") + ":" + "\n"
                    for cr in c.commentrelation_set.all():
                        msg += unicode(cr.content_object) + "\n"
                        msg += 'http://%s%s' % (Site.objects.get_current().domain, cr.content_object.get_absolute_url()) + "\n"
                    
                    msg += "\n"
                    
                    if user.has_perm('public_project.change_comment') and user.email:
                        msg += _("NEW_COMMENT_EMAIL_MESSAGE_ACTIVATION") + "\n"
                        msg += 'http://%s/%s?activation_hash=%s&user=%s' \
                            % (Site.objects.get_current().domain, _("activate_comment_url"), c.activation_hash, urllib.quote_plus(unicode(user))) + "\n"
                    
                    send_mail(subject, msg, settings.EMAIL_FROM, [user.email], fail_silently=False)
            #except AttributeError:
            #    pass
            
            return "SENT"
        else:
            print form.errors
            return "FAILED"
    return ""


def merge_with_search_tag_docs(document_list, object):
    for tag in object.search_tags.all():
        search_docs = search_for_documents(tag.name)
        for doc in search_docs:
            if doc in document_list:
                index = document_list.index(doc)
                if not hasattr(document_list[index], 'search_tags'):
                    document_list[index].search_tags = []
                document_list[index].search_tags.append(tag.name)
            else:
                document_list.append(doc)
    document_list.sort(key=lambda x:x.title)
    return document_list


def index(request):
    research_request_list = ResearchRequest.objects.all()
    comment_list = Comment.objects.filter(published=True)
    
    if Event.objects.count() > 0:
        latest_event = Event.objects.all()[0]
    else:
        latest_event = None
    if Document.objects.count() > 0:
        latest_document = Document.objects.all()[0]
    else:
        latest_document = None 
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='home')[0],
        'current_project_goal_group': ProjectGoalGroup.objects.get_current(),
        'project_part_list': ProjectPart.objects.all(),
        'latest_event': latest_event,
        'latest_document': latest_document,
        'activity_list': ActivityLog.objects.all()[0:5],
        'research_request_list': research_request_list[0:1],
        'num_total_research_requests': len(research_request_list),
        'comment_list': comment_list[0:1],
        'num_total_comments': len(comment_list),
    })
    return render_to_response('index.html', context)


def project_parts(request):
    site_category = SiteCategory.objects.get_or_create(category='project_parts')[0]
    
    main_project_parts = ProjectPart.objects.annotate(count=Count('main_project_parts')).filter(count=0)
    middle = int(math.ceil(float(len(main_project_parts))/float(2)))
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': site_category,
        'main_project_part_list_left': main_project_parts[0:middle],
        'main_project_part_list_right': main_project_parts[middle:],
    })
    return render_to_response('project_parts.html', context)


def project_part(request, project_part_id):
    project_part = get_object_or_404(ProjectPart, pk=project_part_id)
    
    document_list = project_part.get_documents()
    document_list = merge_with_search_tag_docs(list(document_list), project_part)
    
    comment_form_status = validate_comment_form(request)
    
    content_type = ContentType.objects.get(app_label="public_project", model="projectpart")
    comment_list = Comment.objects.filter(commentrelation__content_type=content_type).filter(commentrelation__object_id=project_part.id).filter(published=True).distinct()
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='project_parts')[0],
        'user_comment': get_user_comment(request),
        'project_part': project_part,
        'question_list': project_part.get_questions(),
        'event_list': project_part.get_events(),
        'document_list': document_list,
        'comment_form_status': comment_form_status,
        'comment_list': comment_list[0:3],
        'num_total_comments': len(comment_list),
    })
    return render_to_response('project_part.html', context)


def goals(request):
    
    all_main_project_parts = ProjectPart.objects.annotate(count=Count('main_project_parts')).filter(count=0)
    main_project_parts = []
    for main_pp in all_main_project_parts:
        if main_pp.projectgoalgroup_set.count() > 0:
            main_project_parts.append(main_pp)
    middle = int(math.ceil(float(len(main_project_parts))/float(2)))
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='goals')[0],
        'common_goal_group_list': ProjectGoalGroup.objects.filter(project_part=None).order_by('event'),
        'main_project_part_list_left': main_project_parts[0:middle],
        'main_project_part_list_right': main_project_parts[middle:],
    })
    return render_to_response('goals.html', context)


def questions(request):
    main_project_parts = ProjectPart.objects.annotate(count=Count('main_project_parts')).filter(count=0)
    middle = int(math.ceil(float(len(main_project_parts))/float(2)))
    
    research_request_list = ResearchRequest.objects.all()
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='questions')[0],
        'main_project_part_list_left': main_project_parts[0:middle],
        'main_project_part_list_right': main_project_parts[middle:],
        'research_request_list': research_request_list[0:3],
        'num_total_research_requests': len(research_request_list),
    })
    return render_to_response('questions.html', context)


def question(request, question_id):
    question = get_object_or_404(Question, pk=question_id)
    
    research_request_form_status = validate_research_request_form(request)
    content_type = ContentType.objects.get(app_label="public_project", model="question")
    research_request_list = ResearchRequest.objects.filter(researchrequestrelation__content_type=content_type).filter(researchrequestrelation__object_id=question.id).distinct()
    
    comment_form_status = validate_comment_form(request)
    content_type = ContentType.objects.get(app_label="public_project", model="question")
    comment_list = Comment.objects.filter(commentrelation__content_type=content_type).filter(commentrelation__object_id=question.id).filter(published=True).distinct()
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='questions')[0],
        'user_comment': get_user_comment(request),
        'research_request': get_research_request(request),
        'question': question,
        'research_request_form_status': research_request_form_status,
        'research_request_list': research_request_list[0:3],
        'num_total_research_requests': len(research_request_list),
        'comment_form_status': comment_form_status,
        'comment_list': comment_list[0:3],
        'num_total_comments': len(comment_list),
    })
    return render_to_response('question.html', context)


def participants(request):
    site_category = SiteCategory.objects.get_or_create(category='participants')[0]
    groups = Participant.objects.annotate(count=Count('belongs_to')).filter(count=0)
    middle = int(math.ceil(float(len(groups))/float(2)))
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': site_category,
        'group_list_left': groups[0:middle],
        'group_list_right': groups[middle:],
    })
    return render_to_response('participants.html', context)


def participant(request, participant_id):
    participant = get_object_or_404(Participant, pk=participant_id)
    
    document_list = participant.get_documents()
    document_list = merge_with_search_tag_docs(list(document_list), participant)
    
    comment_form_status = validate_comment_form(request)
    content_type = ContentType.objects.get(app_label="public_project", model="participant")
    comment_list = Comment.objects.filter(commentrelation__content_type=content_type).filter(commentrelation__object_id=participant.id).filter(published=True).distinct()
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='participants')[0],
        'user_comment': get_user_comment(request),
        'participant': participant,
        'question_list': participant.get_questions(),
        'event_list': participant.get_events(),
        'document_list': document_list,
        'comment_form_status': comment_form_status,
        'comment_list': comment_list[0:3],
        'num_total_comments': len(comment_list),
    })
    return render_to_response('participant.html', context)


def events(request):
    site_category = SiteCategory.objects.get_or_create(category='events')[0]
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='events')[0],
        'project_goal_group_list': ProjectGoalGroup.objects.all().order_by('event'),
        'chronology_list': Event.objects.all(),
    })
    return render_to_response('events.html', context)


def event(request, event_id):
    event = get_object_or_404(Event, pk=event_id)
    
    document_list = list(event.related_documents.order_by("title"))
    document_list = merge_with_search_tag_docs(document_list, event)
    
    comment_form_status = validate_comment_form(request)
    content_type = ContentType.objects.get(app_label="public_project", model="event")
    comment_list = Comment.objects.filter(commentrelation__content_type=content_type).filter(commentrelation__object_id=event.id).filter(published=True).distinct()
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': SiteCategory.objects.get_or_create(category='events')[0],
        'user_comment': get_user_comment(request),
        'event': event,
        'document_list': document_list,
        'comment_form_status': comment_form_status,
        'comment_list': comment_list[0:3],
        'num_total_comments': len(comment_list),
    })
    return render_to_response('event.html', context)


def documents(request):
    site_category = SiteCategory.objects.get_or_create(category='documents')[0]
    
    main_project_parts = ProjectPart.objects.annotate(count=Count('main_project_parts')).filter(count=0)
    middle = int(math.ceil(float(len(main_project_parts))/float(2)))
    
    context = RequestContext(request, {
        'site_config': SiteConfig.objects.get_site_config(request),
        'site_category': site_category,
        'main_project_part_list_left': main_project_parts[0:middle],
        'main_project_part_list_right': main_project_parts[middle:],
    })
    return render_to_response('documents.html', context)


def xhr_universal_search(request):
    if request.method == 'GET' and 'query' in request.GET:
        query_string = request.GET['query']
        
        entry_query = get_query(query_string, ['name', 'main_project_part__name',])
        pp_list = list(ProjectPart.objects.select_related().filter(entry_query)[0:10])
        
        entry_query = get_query(query_string, ['title',])
        q_list = list(Question.objects.select_related().filter(entry_query)[0:10])
        
        entry_query = get_query(query_string, ['name',])
        p_list = list(Participant.objects.select_related().filter(entry_query)[0:10])
        
        entry_query = get_query(query_string, ['title',])
        e_list = list(Event.objects.select_related().filter(entry_query)[0:10])
        
        entry_query = get_query(query_string, ['title',])
        d_list = list(Document.objects.select_related().filter(entry_query)[0:10])
        
        object_list = pp_list + q_list + p_list + e_list + d_list
        
        res = {
            'values': {},
            'options': [],
        }
        
        for object in object_list:
            content_type = object.__class__.__name__.lower()
            id = object.id
            
            if not (content_type == request.GET['ommit_content_type'] and str(id) == request.GET['ommit_id']):
                option = '<i class="' + object.get_icon_class() + '"></i> ' + unicode(object)
                res['values'][option] = {
                    'content_type': content_type,
                    'id': id,
                }
                res['options'].append(option)
        
        mimetype = 'application/javascript'
        return HttpResponse(json.dumps(res), mimetype)


def xhr_document_tags(request):
    if request.method == 'POST':
        
        colors = {
            'projectpart': '#0d9434',
            'participant': '#3e3ec7',
            'event': '#c91a1a',
        }
        if request.POST['document_id']:
            document = get_object_or_404(Document, pk=request.POST['document_id'])
            if request.POST['content_type']:
                st_dict_list = SearchTagCacheEntry.objects.filter(document=document).filter(tag__content_type__model=request.POST['content_type']).values('tag__name', 'tag__content_type__model',).annotate(num_tags=Count('tag__name')).order_by('-num_tags')
                #cache_entries = SearchTagCacheEntry.objects.filter(document=document).filter(tag__content_type__model=request.POST['content_type'])
            else:
                st_dict_list = SearchTagCacheEntry.objects.filter(document=document).values('tag__name', 'tag__content_type__model',).annotate(num_tags=Count('tag__name')).order_by('-num_tags')
            st_dict_list = st_dict_list[0:16]
            size_start = 14
            size_span = 10
        else:
            if request.POST['content_type']:
                st_dict_list = SearchTagCacheEntry.objects.filter(tag__content_type__model=request.POST['content_type']).values('tag__name', 'tag__content_type__model',).annotate(num_tags=Count('tag__name')).order_by('-num_tags')[0:22]
            else:
                st_dict_list = SearchTagCacheEntry.objects.values('tag__name', 'tag__content_type__model',).annotate(num_tags=Count('tag__name')).order_by('-num_tags')[0:22]
            size_start = 16
            size_span = 12
        
        if len(st_dict_list) > 0:
            first = st_dict_list[0]
            max_num_results = first['num_tags']
        
        words = []
        for st_dict in st_dict_list:
            text = st_dict['tag__name']
            size = size_start + int(round((float(st_dict['num_tags'])/ float(max_num_results)) * size_span))
            color = colors[st_dict['tag__content_type__model']]
            
            word = { "text": text, "size":size, "color": color, }
            words.append(word)
        
        data = json.dumps(words)
        
        mimetype = 'application/javascript'
        return HttpResponse(data, mimetype)


def document(request, document_id):
    document = get_object_or_404(Document, pk=document_id)
    comment_form_status = validate_comment_form(request)
    
    query_string = ''
    found_pages = []
    if ('q' in request.GET) and request.GET['q'].strip():
        query_string = request.GET['q']
        entry_query = get_query(query_string, ['document__title', 'content',])
        found_pages = Page.objects.select_related().filter(document=document).filter(entry_query).order_by('number')
{'fraction_non_alphanumeric': 0.11948608137044968, 'fraction_numerical': 0.006401442578609264, 'mean_word_length': 2.908554312395384, 'pattern_counts': {'":': 16, '<': 7, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3479545', 'n_tokens_mistral': 13327, 'n_tokens_neox': 12917, 'n_words': 1878}