<commit_before>import datetime
import json

from django.conf import settings
from django.core.paginator import InvalidPage
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.http import Http404, JsonResponse
from django.shortcuts import render, redirect, get_object_or_404
from django.utils.translation import ugettext_lazy as _, activate
from django.views import static
from django.views.decorators.cache import cache_page

from django_hosts.resolvers import reverse

from elasticsearch_dsl import query

from .forms import DocSearchForm
from .models import DocumentRelease, Document
from .search import DocumentDocType, SearchPaginator
from .utils import get_doc_root_or_404, get_doc_path_or_404


CURRENT_LTS = '1.4'
UNSUPPORTED_THRESHOLD = '1.6'
SIMPLE_SEARCH_OPERATORS = ['+', '|', '-', '"', '*', '(', ')', '~']


def version_is_unsupported(version):
    # TODO: would be nice not to hardcode this.
    return version != CURRENT_LTS and version < UNSUPPORTED_THRESHOLD


def index(request):
    return redirect(DocumentRelease.objects.current())


def language(request, lang):
    return redirect(DocumentRelease.objects.current(lang))


def stable(request, lang, version, url):
    path = request.get_full_path()
    current_version = DocumentRelease.objects.current_version()
    return redirect(path.replace(version, current_version, 1))


def document(request, lang, version, url):
    # If either of these can't be encoded as ascii then later on down the line an
    # exception will be emitted by unipath, proactively check for bad data (mostly
    # from the Googlebot) so we can give a nice 404 error.
    try:
        version.encode("ascii")
        url.encode("ascii")
    except UnicodeEncodeError:
        raise Http404

    if lang != 'en':
        activate(lang)

    docroot = get_doc_root_or_404(lang, version)
    doc_path = get_doc_path_or_404(docroot, url)

    if version == 'dev':
        rtd_version = 'latest'
    elif version >= '1.5':
        rtd_version = version + '.x'
    else:
        rtd_version = version + '.X'

    template_names = [
        'docs/%s.html' % docroot.rel_path_to(doc_path).replace(doc_path.ext, ''),
        'docs/doc.html',
    ]
    context = {
        'doc': json.load(open(doc_path, 'rb')),
        'env': json.load(open(docroot.child('globalcontext.json'), 'rb')),
        'lang': lang,
        'version': version,
        'version_is_dev': version == 'dev',
        'version_is_unsupported': version_is_unsupported(version),
        'rtd_version': rtd_version,
        'docurl': url,
        'update_date': datetime.datetime.fromtimestamp(docroot.child('last_build').mtime()),
        'redirect_from': request.GET.get('from', None),
    }
    return render(request, template_names, context)


def pot_file(request, pot_name):
    version = DocumentRelease.objects.current().version
    doc_root = get_doc_root_or_404('en', version, subroot='gettext')
    return static.serve(request, document_root=doc_root, path=pot_name)


def sphinx_static(request, lang, version, path, subpath=None):
    """
    Serve Sphinx static assets from a subdir of the build location.
    """
    document_root = get_doc_root_or_404(lang, version).child(subpath)
    return static.serve(request, document_root=document_root, path=path)


def objects_inventory(request, lang, version):
    response = static.serve(request,
                            document_root=get_doc_root_or_404(lang, version),
                            path="objects.inv")
    response['Content-Type'] = "text/plain"
    return response


def redirect_index(request, *args, **kwargs):
    assert request.path.endswith('index/')
    return redirect(request.path[:-6])


def redirect_search(request):
    """
    Legacy search view to handle old queries correctly, e.g. in scraping
    sites, command line interface etc.
    """
    release = DocumentRelease.objects.current()
    kwargs = {
        'lang': release.lang,
        'version': release.version,
    }
    search_url = reverse('document-search', host='docs', kwargs=kwargs)
    q = request.GET.get('q') or None
    if q:
        search_url += '?q=%s' % q
    return redirect(search_url)


def search_results(request, lang, version, per_page=10, orphans=3):
    """
    Search view to handle language and version specific queries.
    The old search view is being redirected here.
    """
    release = get_object_or_404(DocumentRelease, version=version, lang=lang)
    form = DocSearchForm(request.GET or None, release=release)

    context = {
        'form': form,
        'lang': release.lang,
        'version': release.version,
        'release': release,
        'searchparams': request.GET.urlencode(),
        'version_is_dev': version == 'dev',
        'version_is_unsupported': version_is_unsupported(version),
    }

    if form.is_valid():
        q = form.cleaned_data.get('q')

        if q:
            # catch queries that are coming from browser search bars
            exact = Document.objects.filter(release=release, title=q).first()
            if exact is not None:
                return redirect(exact)
<commit_msg>Use same queryset for doing the exact match to decide weather to redirect or not.<commit_after>import datetime
import json

from django.conf import settings
from django.core.paginator import InvalidPage
from django.contrib.contenttypes.models import ContentType
from django.contrib.sites.models import Site
from django.http import Http404, JsonResponse
from django.shortcuts import render, redirect, get_object_or_404
from django.utils.translation import ugettext_lazy as _, activate
from django.views import static
from django.views.decorators.cache import cache_page

from django_hosts.resolvers import reverse

from elasticsearch_dsl import query

from .forms import DocSearchForm
from .models import DocumentRelease, Document
from .search import DocumentDocType, SearchPaginator
from .utils import get_doc_root_or_404, get_doc_path_or_404


CURRENT_LTS = '1.4'
UNSUPPORTED_THRESHOLD = '1.6'
SIMPLE_SEARCH_OPERATORS = ['+', '|', '-', '"', '*', '(', ')', '~']


def version_is_unsupported(version):
    # TODO: would be nice not to hardcode this.
    return version != CURRENT_LTS and version < UNSUPPORTED_THRESHOLD


def index(request):
    return redirect(DocumentRelease.objects.current())


def language(request, lang):
    return redirect(DocumentRelease.objects.current(lang))


def stable(request, lang, version, url):
    path = request.get_full_path()
    current_version = DocumentRelease.objects.current_version()
    return redirect(path.replace(version, current_version, 1))


def document(request, lang, version, url):
    # If either of these can't be encoded as ascii then later on down the line an
    # exception will be emitted by unipath, proactively check for bad data (mostly
    # from the Googlebot) so we can give a nice 404 error.
    try:
        version.encode("ascii")
        url.encode("ascii")
    except UnicodeEncodeError:
        raise Http404

    if lang != 'en':
        activate(lang)

    docroot = get_doc_root_or_404(lang, version)
    doc_path = get_doc_path_or_404(docroot, url)

    if version == 'dev':
        rtd_version = 'latest'
    elif version >= '1.5':
        rtd_version = version + '.x'
    else:
        rtd_version = version + '.X'

    template_names = [
        'docs/%s.html' % docroot.rel_path_to(doc_path).replace(doc_path.ext, ''),
        'docs/doc.html',
    ]
    context = {
        'doc': json.load(open(doc_path, 'rb')),
        'env': json.load(open(docroot.child('globalcontext.json'), 'rb')),
        'lang': lang,
        'version': version,
        'version_is_dev': version == 'dev',
        'version_is_unsupported': version_is_unsupported(version),
        'rtd_version': rtd_version,
        'docurl': url,
        'update_date': datetime.datetime.fromtimestamp(docroot.child('last_build').mtime()),
        'redirect_from': request.GET.get('from', None),
    }
    return render(request, template_names, context)


def pot_file(request, pot_name):
    version = DocumentRelease.objects.current().version
    doc_root = get_doc_root_or_404('en', version, subroot='gettext')
    return static.serve(request, document_root=doc_root, path=pot_name)


def sphinx_static(request, lang, version, path, subpath=None):
    """
    Serve Sphinx static assets from a subdir of the build location.
    """
    document_root = get_doc_root_or_404(lang, version).child(subpath)
    return static.serve(request, document_root=document_root, path=path)


def objects_inventory(request, lang, version):
    response = static.serve(request,
                            document_root=get_doc_root_or_404(lang, version),
                            path="objects.inv")
    response['Content-Type'] = "text/plain"
    return response


def redirect_index(request, *args, **kwargs):
    assert request.path.endswith('index/')
    return redirect(request.path[:-6])


def redirect_search(request):
    """
    Legacy search view to handle old queries correctly, e.g. in scraping
    sites, command line interface etc.
    """
    release = DocumentRelease.objects.current()
    kwargs = {
        'lang': release.lang,
        'version': release.version,
    }
    search_url = reverse('document-search', host='docs', kwargs=kwargs)
    q = request.GET.get('q') or None
    if q:
        search_url += '?q=%s' % q
    return redirect(search_url)


def search_results(request, lang, version, per_page=10, orphans=3):
    """
    Search view to handle language and version specific queries.
    The old search view is being redirected here.
    """
    release = get_object_or_404(DocumentRelease, version=version, lang=lang)
    form = DocSearchForm(request.GET or None, release=release)

    context = {
        'form': form,
        'lang': release.lang,
        'version': release.version,
        'release': release,
        'searchparams': request.GET.urlencode(),
        'version_is_dev': version == 'dev',
        'version_is_unsupported': version_is_unsupported(version),
    }

    if form.is_valid():
        q = form.cleaned_data.get('q')

        if q:
            # catch queries that are coming from browser search bars
            exact = (DocumentDocType.index_queryset()
                                    .filter(release=release, title=q)
                                    .first())
            if exact is not None:
                return redirect(exact)

{'fraction_non_alphanumeric': 0.11021300874857361, 'fraction_numerical': 0.00893875998478509, 'mean_word_length': 3.3173234811165844, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7409346', 'n_tokens_mistral': 3208, 'n_tokens_neox': 3093, 'n_words': 806}