<commit_before>from os import environ, path, walk
import pkg_resources
import requests
import zipfile

from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import user_passes_test
from django.core.exceptions import PermissionDenied
from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage
from django.core.urlresolvers import reverse
from django.http import (
    JsonResponse,
    HttpResponse,
    HttpResponseNotAllowed,
    Http404,
)
from django.shortcuts import redirect, get_object_or_404, render
from django.utils.http import is_safe_url
from django.utils.translation import (
    LANGUAGE_SESSION_KEY,
    get_language_from_request
)
from django.utils.translation import ugettext as _
from django.views.generic.edit import FormView
from django.views.generic.base import TemplateView
from io import BytesIO
from wagtail.wagtailcore.models import Page, UserPagePermissionsProxy
from wagtail.wagtailsearch.models import Query
from molo.core.utils import generate_slug, get_locale_code, update_media_file
from molo.core.models import (
    ArticlePage, Languages, SiteSettings, Tag,
    ArticlePageTags, SectionPage, ReactionQuestionChoice,
    ReactionQuestionResponse, ReactionQuestion, TranslatablePageMixin)

from molo.core.templatetags.core_tags import get_pages
from molo.core.known_plugins import known_plugins
from molo.core.forms import MediaForm, ReactionQuestionChoiceForm
from molo.core.tasks import copy_to_all_task
from django.views.generic import ListView

from el_pagination.decorators import page_template


def csrf_failure(request, reason=""):
    freebasics_url = settings.FREE_BASICS_URL_FOR_CSRF_MESSAGE
    return render(request, '403_csrf.html', {'freebasics_url': freebasics_url})


def search(request, results_per_page=10, load_more=False):
    search_query = request.GET.get('q', None)
    page = request.GET.get('p', 1)
    locale = get_locale_code(get_language_from_request(request))

    if search_query:
        main = request.site.root_page

        results = ArticlePage.objects.descendant_of(main).filter(
            language__locale=locale
        ).exact_type(ArticlePage).values_list('pk', flat=True)

        # Elasticsearch backend doesn't support filtering
        # on related fields, at the moment.
        # So we need to filter ArticlePage entries using DB,
        # then, we will be able to search
        results = ArticlePage.objects.filter(pk__in=results)
        results = results.live().search(search_query)

        # At the moment only ES backends have highlight API.
        if hasattr(results, 'highlight'):
            results = results.highlight(
                fields={
                    'title': {},
                    'subtitle': {},
                    'body': {},
                },
                require_field_match=False
            )

        Query.get(search_query).add_hit()
    else:
        results = ArticlePage.objects.none()
    if load_more:
        return results
    paginator = Paginator(results, results_per_page)
    try:
        search_results = paginator.page(page)
    except PageNotAnInteger:
        search_results = paginator.page(1)
    except EmptyPage:
        search_results = paginator.page(paginator.num_pages)

    return render(request, 'search/search_results.html', {
        'search_query': search_query,
        'search_results': search_results,
        'results': results,
    })


def locale_set(request, locale):
    request.session[LANGUAGE_SESSION_KEY] = locale
    return redirect(request.GET.get('next', '/'))


def health(request):
    app_id = environ.get('MARATHON_APP_ID', None)
    ver = environ.get('MARATHON_APP_VERSION', None)
    return JsonResponse({'id': app_id, 'version': ver})


def add_translation(request, page_id, locale):
    _page = get_object_or_404(Page, id=page_id)
    page = _page.specific
    if not issubclass(type(page), TranslatablePageMixin):
        messages.add_message(
            request, messages.INFO, _('That page is not translatable.'))
        return redirect(reverse('wagtailadmin_home'))
    # redirect to edit page if translation already exists for this locale
    translated_page = page.translated_pages.filter(language__locale=locale)
    if translated_page.exists():
        return redirect(
            reverse('wagtailadmin_pages:edit', args=[
                translated_page.first().id]))

    # create translation and redirect to edit page
    language = Languages.for_site(request.site).languages.filter(
        locale=locale).first()
    if not language:
        raise Http404
    new_title = str(language) + " translation of %s" % page.title
    new_slug = generate_slug(new_title)
    translation = page.__class__(
        title=new_title, slug=new_slug, language=language)
    translation.save_revision()
    page.get_parent().add_child(instance=translation)
    # add the translation the new way
    page.specific.translated_pages.add(translation)
    page.save()
    for translated_page in \
            page.specific.translated_pages.all():
        translations = page.specific.translated_pages.all().\
            exclude(language__pk=translated_page.language.pk)
        for translation in translations:
            translated_page.translated_pages.add(translation)
        translated_page.save()

    # make sure new translation is in draft mode
    translation.unpublish()
    return redirect(
        reverse('wagtailadmin_pages:edit', args=[translation.id]))


def versions(request):
    comparison_url = "https://github.com/praekelt/%s/compare/%s...%s"
    plugins_info = []
    for plugin in known_plugins():
        try:<commit_msg>save page after instance added to parent<commit_after>from os import environ, path, walk
import pkg_resources
import requests
import zipfile

from django.conf import settings
from django.contrib import messages
from django.contrib.auth.decorators import user_passes_test
from django.core.exceptions import PermissionDenied
from django.core.paginator import Paginator, PageNotAnInteger, EmptyPage
from django.core.urlresolvers import reverse
from django.http import (
    JsonResponse,
    HttpResponse,
    HttpResponseNotAllowed,
    Http404,
)
from django.shortcuts import redirect, get_object_or_404, render
from django.utils.http import is_safe_url
from django.utils.translation import (
    LANGUAGE_SESSION_KEY,
    get_language_from_request
)
from django.utils.translation import ugettext as _
from django.views.generic.edit import FormView
from django.views.generic.base import TemplateView
from io import BytesIO
from wagtail.wagtailcore.models import Page, UserPagePermissionsProxy
from wagtail.wagtailsearch.models import Query
from molo.core.utils import generate_slug, get_locale_code, update_media_file
from molo.core.models import (
    ArticlePage, Languages, SiteSettings, Tag,
    ArticlePageTags, SectionPage, ReactionQuestionChoice,
    ReactionQuestionResponse, ReactionQuestion, TranslatablePageMixin)

from molo.core.templatetags.core_tags import get_pages
from molo.core.known_plugins import known_plugins
from molo.core.forms import MediaForm, ReactionQuestionChoiceForm
from molo.core.tasks import copy_to_all_task
from django.views.generic import ListView

from el_pagination.decorators import page_template


def csrf_failure(request, reason=""):
    freebasics_url = settings.FREE_BASICS_URL_FOR_CSRF_MESSAGE
    return render(request, '403_csrf.html', {'freebasics_url': freebasics_url})


def search(request, results_per_page=10, load_more=False):
    search_query = request.GET.get('q', None)
    page = request.GET.get('p', 1)
    locale = get_locale_code(get_language_from_request(request))

    if search_query:
        main = request.site.root_page

        results = ArticlePage.objects.descendant_of(main).filter(
            language__locale=locale
        ).exact_type(ArticlePage).values_list('pk', flat=True)

        # Elasticsearch backend doesn't support filtering
        # on related fields, at the moment.
        # So we need to filter ArticlePage entries using DB,
        # then, we will be able to search
        results = ArticlePage.objects.filter(pk__in=results)
        results = results.live().search(search_query)

        # At the moment only ES backends have highlight API.
        if hasattr(results, 'highlight'):
            results = results.highlight(
                fields={
                    'title': {},
                    'subtitle': {},
                    'body': {},
                },
                require_field_match=False
            )

        Query.get(search_query).add_hit()
    else:
        results = ArticlePage.objects.none()
    if load_more:
        return results
    paginator = Paginator(results, results_per_page)
    try:
        search_results = paginator.page(page)
    except PageNotAnInteger:
        search_results = paginator.page(1)
    except EmptyPage:
        search_results = paginator.page(paginator.num_pages)

    return render(request, 'search/search_results.html', {
        'search_query': search_query,
        'search_results': search_results,
        'results': results,
    })


def locale_set(request, locale):
    request.session[LANGUAGE_SESSION_KEY] = locale
    return redirect(request.GET.get('next', '/'))


def health(request):
    app_id = environ.get('MARATHON_APP_ID', None)
    ver = environ.get('MARATHON_APP_VERSION', None)
    return JsonResponse({'id': app_id, 'version': ver})


def add_translation(request, page_id, locale):
    _page = get_object_or_404(Page, id=page_id)
    page = _page.specific
    if not issubclass(type(page), TranslatablePageMixin):
        messages.add_message(
            request, messages.INFO, _('That page is not translatable.'))
        return redirect(reverse('wagtailadmin_home'))
    # redirect to edit page if translation already exists for this locale
    translated_page = page.translated_pages.filter(language__locale=locale)
    if translated_page.exists():
        return redirect(
            reverse('wagtailadmin_pages:edit', args=[
                translated_page.first().id]))

    # create translation and redirect to edit page
    language = Languages.for_site(request.site).languages.filter(
        locale=locale).first()
    if not language:
        raise Http404
    new_title = str(language) + " translation of %s" % page.title
    new_slug = generate_slug(new_title)
    translation = page.__class__(
        title=new_title, slug=new_slug, language=language)
    page.get_parent().add_child(instance=translation)
    translation.save_revision()
    # add the translation the new way
    page.specific.translated_pages.add(translation)
    page.save()
    for translated_page in \
            page.specific.translated_pages.all():
        translations = page.specific.translated_pages.all().\
            exclude(language__pk=translated_page.language.pk)
        for translation in translations:
            translated_page.translated_pages.add(translation)
        translated_page.save()

    # make sure new translation is in draft mode
    translation.unpublish()
    return redirect(
        reverse('wagtailadmin_pages:edit', args=[translation.id]))


def versions(request):
    comparison_url = "https://github.com/praekelt/%s/compare/%s...%s"
    plugins_info = []
    for plugin in known_plugins():
        try:
{'fraction_non_alphanumeric': 0.08727719729563614, 'fraction_numerical': 0.0033365528141188865, 'mean_word_length': 3.5001975503753457, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3053892', 'n_tokens_mistral': 3279, 'n_tokens_neox': 3180, 'n_words': 763}