<commit_before>import logging
import time
from datetime import datetime
from difflib import SequenceMatcher

import requests

from django.conf import settings
from django.contrib import admin
from django.core.exceptions import PermissionDenied
from django.http import HttpResponseRedirect, Http404
from django.shortcuts import render

from kitsune.search import synonym_utils
from kitsune.search.es_utils import (
    get_doctype_stats, get_indexes, delete_index, ES_EXCEPTIONS,
    get_indexable, CHUNK_SIZE, recreate_indexes, write_index, read_index,
    all_read_indexes, all_write_indexes)
from kitsune.search.models import Record, get_mapping_types, Synonym
from kitsune.search.tasks import (
    index_chunk_task, reconcile_task, update_synonyms_task)
from kitsune.search.utils import chunked, to_class_path
from kitsune.wiki.models import Document, DocumentMappingType


log = logging.getLogger('k.es')


def handle_reset(request):
    """Resets records"""
    for rec in Record.objects.outstanding():
        rec.mark_fail('Cancelled.')

    return HttpResponseRedirect(request.path)


class DeleteError(Exception):
    pass


def create_batch_id():
    """Returns a batch_id"""
    # TODO: This is silly, but it's a good enough way to distinguish
    # between batches by looking at a Record. This is just over the
    # number of seconds in a day.
    return str(int(time.time()))[-6:]


def handle_reconcile(request):
    """Reconcile all the things"""
    outstanding = Record.objects.outstanding().count()
    if outstanding > 0:
        raise ReindexError('There are %s outstanding chunks.' % outstanding)

    batch_id = create_batch_id()

    for cls, indexable in get_indexable():
        index = cls.get_index()
        doc_type = cls.get_mapping_type_name()
        chunk_name = 'Reconciling: %s' % doc_type
        rec = Record(batch_id=batch_id, name=chunk_name)
        rec.save()
        reconcile_task.delay(index, batch_id, rec.id, doc_type)

    return HttpResponseRedirect(request.path)


def handle_delete(request):
    """Deletes an index"""
    index_to_delete = request.POST.get('delete_index')
    es_indexes = [name for (name, count) in get_indexes()]

    # Rule 1: Has to start with the ES_INDEX_PREFIX.
    if not index_to_delete.startswith(settings.ES_INDEX_PREFIX):
        raise DeleteError('"%s" is not a valid index name.' % index_to_delete)

    # Rule 2: Must be an existing index.
    if index_to_delete not in es_indexes:
        raise DeleteError('"%s" does not exist.' % index_to_delete)

    # Rule 3: Don't delete the default read index.
    # TODO: When the critical index exists, this should be "Don't
    # delete the critical read index."
    if index_to_delete == read_index('default'):
        raise DeleteError('"%s" is the default read index.' % index_to_delete)

    # The index is ok to delete
    delete_index(index_to_delete)

    return HttpResponseRedirect(request.path)


class ReindexError(Exception):
    pass


def reindex(mapping_type_names):
    """Reindex all instances of a given mapping type with celery tasks

    :arg mapping_type_names: list of mapping types to reindex

    """
    outstanding = Record.objects.outstanding().count()
    if outstanding > 0:
        raise ReindexError('There are %s outstanding chunks.' % outstanding)

    batch_id = create_batch_id()

    # Break up all the things we want to index into chunks. This
    # chunkifies by class then by chunk size. Also generate
    # reconcile_tasks.
    chunks = []
    for cls, indexable in get_indexable(mapping_types=mapping_type_names):
        chunks.extend(
            (cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))

        index = cls.get_index()
        doc_type = cls.get_mapping_type_name()
        chunk_name = 'Reconciling: %s' % doc_type
        rec = Record(batch_id=batch_id, name=chunk_name)
        rec.save()
        reconcile_task.delay(index, batch_id, rec.id, doc_type)

    for cls, id_list in chunks:
        index = cls.get_index()
        chunk_name = 'Indexing: %s %d -> %d' % (
            cls.get_mapping_type_name(), id_list[0], id_list[-1])
        rec = Record(batch_id=batch_id, name=chunk_name)
        rec.save()
        index_chunk_task.delay(index, batch_id, rec.id, (to_class_path(cls), id_list))


def handle_recreate_index(request):
    """Deletes an index, recreates it, and reindexes it."""
    groups = [name.replace('check_', '')
              for name in request.POST.keys()
              if name.startswith('check_')]

    indexes = [write_index(group) for group in groups]
    recreate_indexes(indexes=indexes)

    mapping_types_names = [mt.get_mapping_type_name()
                           for mt in get_mapping_types()
                           if mt.get_index_group() in groups]
    reindex(mapping_types_names)

    return HttpResponseRedirect(request.path)


def handle_reindex(request):
    """Caculates and kicks off indexing tasks"""
    mapping_type_names = [name.replace('check_', '')
                          for name in request.POST.keys()
                          if name.startswith('check_')]

    reindex(mapping_type_names)

    return HttpResponseRedirect(request.path)<commit_msg>[bug 917754] Convert to use Record.objects.create<commit_after>import logging
import time
from datetime import datetime
from difflib import SequenceMatcher

import requests

from django.conf import settings
from django.contrib import admin
from django.core.exceptions import PermissionDenied
from django.http import HttpResponseRedirect, Http404
from django.shortcuts import render

from kitsune.search import synonym_utils
from kitsune.search.es_utils import (
    get_doctype_stats, get_indexes, delete_index, ES_EXCEPTIONS,
    get_indexable, CHUNK_SIZE, recreate_indexes, write_index, read_index,
    all_read_indexes, all_write_indexes)
from kitsune.search.models import Record, get_mapping_types, Synonym
from kitsune.search.tasks import (
    index_chunk_task, reconcile_task, update_synonyms_task)
from kitsune.search.utils import chunked, to_class_path
from kitsune.wiki.models import Document, DocumentMappingType


log = logging.getLogger('k.es')


def handle_reset(request):
    """Resets records"""
    for rec in Record.objects.outstanding():
        rec.mark_fail('Cancelled.')

    return HttpResponseRedirect(request.path)


class DeleteError(Exception):
    pass


def create_batch_id():
    """Returns a batch_id"""
    # TODO: This is silly, but it's a good enough way to distinguish
    # between batches by looking at a Record. This is just over the
    # number of seconds in a day.
    return str(int(time.time()))[-6:]


def handle_reconcile(request):
    """Reconcile all the things"""
    outstanding = Record.objects.outstanding().count()
    if outstanding > 0:
        raise ReindexError('There are %s outstanding chunks.' % outstanding)

    batch_id = create_batch_id()

    for cls, indexable in get_indexable():
        index = cls.get_index()
        doc_type = cls.get_mapping_type_name()
        chunk_name = 'Reconciling: %s' % doc_type
        rec = Record.objects.create(batch_id=batch_id, name=chunk_name)
        reconcile_task.delay(index, batch_id, rec.id, doc_type)

    return HttpResponseRedirect(request.path)


def handle_delete(request):
    """Deletes an index"""
    index_to_delete = request.POST.get('delete_index')
    es_indexes = [name for (name, count) in get_indexes()]

    # Rule 1: Has to start with the ES_INDEX_PREFIX.
    if not index_to_delete.startswith(settings.ES_INDEX_PREFIX):
        raise DeleteError('"%s" is not a valid index name.' % index_to_delete)

    # Rule 2: Must be an existing index.
    if index_to_delete not in es_indexes:
        raise DeleteError('"%s" does not exist.' % index_to_delete)

    # Rule 3: Don't delete the default read index.
    # TODO: When the critical index exists, this should be "Don't
    # delete the critical read index."
    if index_to_delete == read_index('default'):
        raise DeleteError('"%s" is the default read index.' % index_to_delete)

    # The index is ok to delete
    delete_index(index_to_delete)

    return HttpResponseRedirect(request.path)


class ReindexError(Exception):
    pass


def reindex(mapping_type_names):
    """Reindex all instances of a given mapping type with celery tasks

    :arg mapping_type_names: list of mapping types to reindex

    """
    outstanding = Record.objects.outstanding().count()
    if outstanding > 0:
        raise ReindexError('There are %s outstanding chunks.' % outstanding)

    batch_id = create_batch_id()

    # Break up all the things we want to index into chunks. This
    # chunkifies by class then by chunk size. Also generate
    # reconcile_tasks.
    chunks = []
    for cls, indexable in get_indexable(mapping_types=mapping_type_names):
        chunks.extend(
            (cls, chunk) for chunk in chunked(indexable, CHUNK_SIZE))

        index = cls.get_index()
        doc_type = cls.get_mapping_type_name()
        chunk_name = 'Reconciling: %s' % doc_type
        rec = Record.objects.create(batch_id=batch_id, name=chunk_name)
        reconcile_task.delay(index, batch_id, rec.id, doc_type)

    for cls, id_list in chunks:
        index = cls.get_index()
        chunk_name = 'Indexing: %s %d -> %d' % (
            cls.get_mapping_type_name(), id_list[0], id_list[-1])
        rec = Record.objects.create(batch_id=batch_id, name=chunk_name)
        index_chunk_task.delay(index, batch_id, rec.id, (to_class_path(cls), id_list))


def handle_recreate_index(request):
    """Deletes an index, recreates it, and reindexes it."""
    groups = [name.replace('check_', '')
              for name in request.POST.keys()
              if name.startswith('check_')]

    indexes = [write_index(group) for group in groups]
    recreate_indexes(indexes=indexes)

    mapping_types_names = [mt.get_mapping_type_name()
                           for mt in get_mapping_types()
                           if mt.get_index_group() in groups]
    reindex(mapping_types_names)

    return HttpResponseRedirect(request.path)


def handle_reindex(request):
    """Caculates and kicks off indexing tasks"""
    mapping_type_names = [name.replace('check_', '')
                          for name in request.POST.keys()
                          if name.startswith('check_')]

    reindex(mapping_type_names)

    return HttpResponseRedirect(request.path)
{'fraction_non_alphanumeric': 0.092486103124401, 'fraction_numerical': 0.0026835345984282154, 'mean_word_length': 3.3992411467116357, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6238565', 'n_tokens_mistral': 3198, 'n_tokens_neox': 3107, 'n_words': 875}