<commit_before>"""Program reindex"""

import pytz
import json
import requests
import logging
from datetime import datetime

from django.http import HttpResponse, JsonResponse
from django.contrib.auth.decorators import user_passes_test
from django.shortcuts import redirect
from django.core.urlresolvers import reverse

from opaque_keys.edx.keys import CourseKey
from search.search_engine_base import SearchEngine
from .models import Program
log = logging.getLogger('edx.modulestore')


@user_passes_test(lambda u: u.is_superuser)
def index_programs_information(request):
    """
    Add all the program to the course discovery index

    """
    INDEX_NAME = "courseware_index"
    searcher = SearchEngine.get_search_engine(INDEX_NAME)
    if not searcher:
        return

    programs = Program.objects.all()
    for program in programs:
        if program.start <= datetime.now(pytz.UTC).date():
            program_info = {
                'id': program.id,
                'course': program.id,
                'content': {
                    'display_name': program.name,
                    'overview': program.overview
                },
                'image_url': program.banner_image.url,
                'start': program.start,
                'language': program.language.code,
                'subject': program.subject.name,
                'is_program': True,
            }
        # Broad exception handler to protect around and report problems with indexing
            try:
                searcher.index('course_info', [program_info])
            except:  # pylint: disable=bare-except
                log.exception(
                    "Program discovery indexing error encountered %s",
                    program_info.get('id', ''),
                )
                return JsonResponse({
                    'success': False,
                    'error': 'Program discovery indexing error encountered ',
                    'Program': program.name
                })
        else:
            return JsonResponse({
                    'success': False,
                    'error': 'This ' + program.name + ' program start date is in future: ' + str(program.start),
                    'suggestion': 'Please set past progrm start date',
                    'Program': program.name
                })

    return redirect(reverse('show-programs'))


def index_course_programs(course_id):
    """
    reindex only the program that containts course

    """
    INDEX_NAME = "courseware_index"
    searcher = SearchEngine.get_search_engine(INDEX_NAME)
    if not searcher:
        return
    course_key = CourseKey.from_string(course_id)
    programs = Program.objects.filter(courses__course_key=course_key)
    for program in programs:
        if program.start <= datetime.now(pytz.UTC).date():
            program_info = {
                'id': program.id,
                'course': program.id,
                'content': {
                    'display_name': program.name,
                    'overview': program.overview
                },
                'image_url': program.banner_image.url,
                'start': program.start,
                'language': program.language.code,
                'subject': program.subject.name,
                'is_program': True,
            }
        # Broad exception handler to protect around and report problems with indexing
            try:
                searcher.index('course_info', [program_info])
            except:  # pylint: disable=bare-except
                log.exception(
                    "Program discovery indexing error encountered %s",
                    program_info.get('id', ''),
                )
                return JsonResponse({
                    'success': False,
                    'error': 'Program discovery indexing error encountered ',
                    'Program': program.name
                })
        else:
            return JsonResponse({
                    'success': False,
                    'error': 'This ' + program.name + ' program start date is in future: ' + str(program.start),
                    'suggestion': 'Please set past progrm start date',
                    'Program': program.name
                })
    return True


@user_passes_test(lambda u: u.is_superuser)
def reindex_specific_program(request, pk):
    """
    reindex specific program

    """
    INDEX_NAME = "courseware_index"
    searcher = SearchEngine.get_search_engine(INDEX_NAME)
    if not searcher:
        return

    try:
        program = Program.objects.get(pk=pk)
    except Exception, e:
        return JsonResponse({
                'success': False,
                'error': 'Program not found in this id ' + pk,
            })

    if program.start <= datetime.now(pytz.UTC).date():
        program_info = {
            'id': program.id,
            'course': program.id,
            'content': {
                'display_name': program.name,
                'overview': program.overview
            },
            'image_url': program.banner_image.url,
            'start': program.start,
            'language': program.language.code,
            'subject': program.subject.name,<commit_msg>add short overview of programs<commit_after>"""Program reindex"""

import pytz
import json
import requests
import logging
from datetime import datetime

from django.http import HttpResponse, JsonResponse
from django.contrib.auth.decorators import user_passes_test
from django.shortcuts import redirect
from django.core.urlresolvers import reverse

from opaque_keys.edx.keys import CourseKey
from search.search_engine_base import SearchEngine
from .models import Program
log = logging.getLogger('edx.modulestore')


@user_passes_test(lambda u: u.is_superuser)
def index_programs_information(request):
    """
    Add all the program to the course discovery index

    """
    INDEX_NAME = "courseware_index"
    searcher = SearchEngine.get_search_engine(INDEX_NAME)
    if not searcher:
        return

    programs = Program.objects.all()
    for program in programs:
        if program.start <= datetime.now(pytz.UTC).date():
            program_info = {
                'id': program.id,
                'course': program.id,
                'content': {
                    'display_name': program.name,
                    'overview': program.short_description
                },
                'image_url': program.banner_image.url,
                'start': program.start,
                'language': program.language.code,
                'subject': program.subject.name,
                'is_program': True,
            }
        # Broad exception handler to protect around and report problems with indexing
            try:
                searcher.index('course_info', [program_info])
            except:  # pylint: disable=bare-except
                log.exception(
                    "Program discovery indexing error encountered %s",
                    program_info.get('id', ''),
                )
                return JsonResponse({
                    'success': False,
                    'error': 'Program discovery indexing error encountered ',
                    'Program': program.name
                })
        else:
            return JsonResponse({
                    'success': False,
                    'error': 'This ' + program.name + ' program start date is in future: ' + str(program.start),
                    'suggestion': 'Please set past progrm start date',
                    'Program': program.name
                })

    return redirect(reverse('show-programs'))


def index_course_programs(course_id):
    """
    reindex only the program that containts course

    """
    INDEX_NAME = "courseware_index"
    searcher = SearchEngine.get_search_engine(INDEX_NAME)
    if not searcher:
        return
    course_key = CourseKey.from_string(course_id)
    programs = Program.objects.filter(courses__course_key=course_key)
    for program in programs:
        if program.start <= datetime.now(pytz.UTC).date():
            program_info = {
                'id': program.id,
                'course': program.id,
                'content': {
                    'display_name': program.name,
                    'overview': program.short_description
                },
                'image_url': program.banner_image.url,
                'start': program.start,
                'language': program.language.code,
                'subject': program.subject.name,
                'is_program': True,
            }
        # Broad exception handler to protect around and report problems with indexing
            try:
                searcher.index('course_info', [program_info])
            except:  # pylint: disable=bare-except
                log.exception(
                    "Program discovery indexing error encountered %s",
                    program_info.get('id', ''),
                )
                return JsonResponse({
                    'success': False,
                    'error': 'Program discovery indexing error encountered ',
                    'Program': program.name
                })
        else:
            return JsonResponse({
                    'success': False,
                    'error': 'This ' + program.name + ' program start date is in future: ' + str(program.start),
                    'suggestion': 'Please set past progrm start date',
                    'Program': program.name
                })
    return True


@user_passes_test(lambda u: u.is_superuser)
def reindex_specific_program(request, pk):
    """
    reindex specific program

    """
    INDEX_NAME = "courseware_index"
    searcher = SearchEngine.get_search_engine(INDEX_NAME)
    if not searcher:
        return

    try:
        program = Program.objects.get(pk=pk)
    except Exception, e:
        return JsonResponse({
                'success': False,
                'error': 'Program not found in this id ' + pk,
            })

    if program.start <= datetime.now(pytz.UTC).date():
        program_info = {
            'id': program.id,
            'course': program.id,
            'content': {
                'display_name': program.name,
                'overview': program.short_description
            },
            'image_url': program.banner_image.url,
            'start': program.start,
            'language': program.language.code,
            'subject': program.subject.name,
{'fraction_non_alphanumeric': 0.09530175706646295, 'fraction_numerical': 0.0, 'mean_word_length': 1.7337509788566954, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6740264', 'n_tokens_mistral': 2720, 'n_tokens_neox': 2600, 'n_words': 655}