<commit_before># -*- coding: utf-8 -*-
# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © <NAME>. See NOTICE for more information.
#
import tempfile
import zipfile
import sys
import pprint
import traceback
import re

from django.conf import settings
from django.template import RequestContext
from django.shortcuts import render_to_response, get_object_or_404
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.contrib.auth.decorators import login_required, user_passes_test
from django.utils.datastructures import SortedDict
from django.views.decorators.http import require_POST
from django.contrib import auth
from django.contrib.auth.forms import UserCreationForm, AuthenticationForm
from django.utils import simplejson
from django.utils.functional import Promise
from django.utils.encoding import force_unicode
from django.utils.http import urlquote_plus
from django.views.decorators import cache
from django.utils.translation import ugettext as _

from catalogue import models
from catalogue import forms
from catalogue.utils import split_tags
from newtagging import views as newtagging_views


staff_required = user_passes_test(lambda user: user.is_staff)


class LazyEncoder(simplejson.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Promise):
            return force_unicode(obj)
        return obj


def main_page(request):    
    if request.user.is_authenticated():
        shelves = models.Tag.objects.filter(category='set', user=request.user)
        new_set_form = forms.NewSetForm()
    extra_where = "NOT catalogue_tag.category = 'set'"
    tags = models.Tag.objects.usage_for_model(models.Book, counts=True, extra={'where': [extra_where]})
    fragment_tags = models.Tag.objects.usage_for_model(models.Fragment, counts=True,
        extra={'where': ["catalogue_tag.category = 'theme'"] + [extra_where]})
    categories = split_tags(tags)
    
    form = forms.SearchForm()
    return render_to_response('catalogue/main_page.html', locals(),
        context_instance=RequestContext(request))


def book_list(request):
    books = models.Book.objects.all()
    form = forms.SearchForm()
    
    books_by_first_letter = SortedDict()
    for book in books:
        books_by_first_letter.setdefault(book.title[0], []).append(book)
    
    return render_to_response('catalogue/book_list.html', locals(),
        context_instance=RequestContext(request))


def tagged_object_list(request, tags=''):
    # Prevent DoS attacks on our database
    if len(tags.split('/')) > 6:
        raise Http404
        
    try:
        tags = models.Tag.get_tag_list(tags)
    except models.Tag.DoesNotExist:
        raise Http404
    
    if len([tag for tag in tags if tag.category == 'book']):
        raise Http404
    
    model = models.Book
    shelf = [tag for tag in tags if tag.category == 'set']
    shelf_is_set = (len(tags) == 1 and tags[0].category == 'set')
    theme_is_set = len([tag for tag in tags if tag.category == 'theme']) > 0
    if theme_is_set:
        model = models.Fragment
    only_author = len(tags) == 1 and tags[0].category == 'author'
    pd_counter = only_author and tags[0].goes_to_pd()

    user_is_owner = (len(shelf) and request.user.is_authenticated() and request.user == shelf[0].user)
    
    extra_where = "catalogue_tag.category NOT IN ('set', 'book')"
    related_tags = models.Tag.objects.related_for_model(tags, model, counts=True, extra={'where': [extra_where]})
    categories = split_tags(related_tags)

    if not (theme_is_set or shelf_is_set):
        model=models.Book.objects.filter(parent=None)
    
    return newtagging_views.tagged_object_list(
        request,
        tag_model=models.Tag,
        queryset_or_model=model,
        tags=tags,
        template_name='catalogue/tagged_object_list.html',
        extra_context = {
            'categories': categories,
            'shelf_is_set': shelf_is_set,
            'only_author': only_author,
            'pd_counter': pd_counter,
            'user_is_owner': user_is_owner,
            'formats_form': forms.DownloadFormatsForm(),
        },
    )


def book_fragments(request, book_slug, theme_slug):
    book = get_object_or_404(models.Book, slug=book_slug)
    book_tag = get_object_or_404(models.Tag, slug='l-' + book_slug)
    theme = get_object_or_404(models.Tag, slug=theme_slug)
    fragments = models.Fragment.tagged.with_all([book_tag, theme])
    
    form = forms.SearchForm()
    return render_to_response('catalogue/book_fragments.html', locals(),
        context_instance=RequestContext(request))


def book_detail(request, slug):
    try:
        book = models.Book.objects.get(slug=slug)
    except models.Book.DoesNotExist:
        return book_stub_detail(request, slug)

    book_tag = get_object_or_404(models.Tag, slug = 'l-' + slug)
    tags = list(book.tags.filter(~Q(category='set')))
    categories = split_tags(tags)<commit_msg>fixed theme counts (#318, #319)<commit_after># -*- coding: utf-8 -*-
# This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
# Copyright © <NAME>. See NOTICE for more information.
#
import tempfile
import zipfile
import sys
import pprint
import traceback
import re

from django.conf import settings
from django.template import RequestContext
from django.shortcuts import render_to_response, get_object_or_404
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.core.urlresolvers import reverse
from django.db.models import Q
from django.contrib.auth.decorators import login_required, user_passes_test
from django.utils.datastructures import SortedDict
from django.views.decorators.http import require_POST
from django.contrib import auth
from django.contrib.auth.forms import UserCreationForm, AuthenticationForm
from django.utils import simplejson
from django.utils.functional import Promise
from django.utils.encoding import force_unicode
from django.utils.http import urlquote_plus
from django.views.decorators import cache
from django.utils.translation import ugettext as _

from catalogue import models
from catalogue import forms
from catalogue.utils import split_tags
from newtagging import views as newtagging_views


staff_required = user_passes_test(lambda user: user.is_staff)


class LazyEncoder(simplejson.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, Promise):
            return force_unicode(obj)
        return obj


def main_page(request):    
    if request.user.is_authenticated():
        shelves = models.Tag.objects.filter(category='set', user=request.user)
        new_set_form = forms.NewSetForm()
    extra_where = "NOT catalogue_tag.category = 'set'"
    tags = models.Tag.objects.usage_for_model(models.Book, counts=True, extra={'where': [extra_where]})
    fragment_tags = models.Tag.objects.usage_for_model(models.Fragment, counts=True,
        extra={'where': ["catalogue_tag.category = 'theme'"] + [extra_where]})
    categories = split_tags(tags)
    
    form = forms.SearchForm()
    return render_to_response('catalogue/main_page.html', locals(),
        context_instance=RequestContext(request))


def book_list(request):
    books = models.Book.objects.all()
    form = forms.SearchForm()
    
    books_by_first_letter = SortedDict()
    for book in books:
        books_by_first_letter.setdefault(book.title[0], []).append(book)
    
    return render_to_response('catalogue/book_list.html', locals(),
        context_instance=RequestContext(request))


def tagged_object_list(request, tags=''):
    # Prevent DoS attacks on our database
    if len(tags.split('/')) > 6:
        raise Http404
        
    try:
        tags = models.Tag.get_tag_list(tags)
    except models.Tag.DoesNotExist:
        raise Http404
    
    if len([tag for tag in tags if tag.category == 'book']):
        raise Http404
    
    theme_is_set = [tag for tag in tags if tag.category == 'theme']
    shelf_is_set = len(tags) == 1 and tags[0].category == 'set'
    my_shelf_is_set = shelf_is_set and request.user.is_authenticated() and request.user == shelf[0].user
    
    objects = only_author = pd_counter = categories = None
    
    if theme_is_set:
        shelf_tags = [tag for tag in tags if tag.category == 'set']
        fragment_tags = [tag for tag in tags if tag.category != 'set']
        fragments = models.Fragment.tagged.with_all(fragment_tags)
        
        if shelf_tags:
            books = models.Book.tagged.with_all(shelf_tags)
            l_tags = [models.Tag.objects.get(slug = 'l-' + book.slug) for book in books]
            fragments = (fragment for fragment in models.Fragment.tagged.with_any(l_tags) if fragment in fragments)
        
        fragment_keys = (fragment.pk for fragment in fragments)
        if fragment_keys:
            related_tags = models.Fragment.tags.usage(counts=True,
                                filters={'pk__in': fragment_keys}, 
                                extra={'where': ["catalogue_tag.category != 'book'"]})
            related_tags = (tag for tag in related_tags if tag not in fragment_tags)
            categories = split_tags(related_tags)
            
            objects = fragments
    else:
        books = models.Book.tagged.with_all(tags)
        l_tags = [models.Tag.objects.get(slug = 'l-' + book.slug) for book in books]
        book_keys = [book.pk for book in books]
        if book_keys:
            related_tags = models.Book.tags.usage(counts=True,
                                filters={'pk__in': book_keys}, 
                                extra={'where': ["catalogue_tag.category NOT IN ('set', 'book', 'theme')"]})
            categories = split_tags(related_tags)
    
            fragment_keys = [fragment.pk for fragment in models.Fragment.tagged.with_any(l_tags)]
            if fragment_keys:
                categories['theme'] = models.Fragment.tags.usage(counts=True,
                                    filters={'pk__in': fragment_keys}, 
                                    extra={'where': ["catalogue_tag.category = 'theme'"]})
                
            books = books.exclude(parent__in = book_keys)
            objects = books
        
    if not objects:
        only_author = len(tags) == 1 and tags[0].category == 'author'
        pd_counter = only_author and tags[0].goes_to_pd()
        objects = models.Book.objects.none()
    
    return newtagging_views.tagged_object_list(
        request,
        tag_model=models.Tag,
        queryset_or_model=objects,
        tags=tags,
        template_name='catalogue/tagged_object_list.html',
        extra_context = {
            'categories': categories,
            'shelf_is_set': shelf_is_set,
            'only_author': only_author,
            'pd_counter': pd_counter,
            'user_is_owner': my_shelf_is_set,
            'formats_form': forms.DownloadFormatsForm(),
        },
    )


def book_fragments(request, book_slug, theme_slug):
    book = get_object_or_404(models.Book, slug=book_slug)
    book_tag = get_object_or_404(models.Tag, slug='l-' + book_slug)
    theme = get_object_or_404(models.Tag, slug=theme_slug)
    fragments = models.Fragment.tagged.with_all([book_tag, theme])
    
    form = forms.SearchForm()
    return render_to_response('catalogue/book_fragments.html', locals(),
        context_instance=RequestContext(request))


def book_detail(request, slug):
    try:
        book = models.Book.objects.get(slug=slug)
    except models.Book.DoesNotExist:
        return book_stub_detail(request, slug)

    book_tag = get_object_or_404(models.Tag, slug = 'l-' + slug)
    tags = list(book.tags.filter(~Q(category='set')))
    categories = split_tags(tags)
{'fraction_non_alphanumeric': 0.1003310415075121, 'fraction_numerical': 0.006875477463712758, 'mean_word_length': 3.2274847506279154, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1841474', 'n_tokens_mistral': 3581, 'n_tokens_neox': 3477, 'n_words': 819}