<commit_before>import logging
from datetime import datetime

from django.contrib import admin
from django.http import HttpResponseRedirect
from django.shortcuts import render

from pyelasticsearch.exceptions import (
    ConnectionError, ElasticHttpNotFoundError, Timeout)

from fjord.search.index import (
    chunked, get_index, get_index_stats, get_indexes, get_indexable,
    recreate_index, create_batch_id)
from fjord.search.models import Record
from fjord.search.tasks import index_chunk_task


log = logging.getLogger('i.search')


CHUNK_SIZE = 50000


def handle_reset(request):
    """Mark outstanding Records as failed.

    Why? You'd want to reset the system if it gets itself wedged
    thinking there are outstanding tasks, but there aren't. This
    lets you fix that.

    """
    for rec in Record.outstanding():
        rec.mark_fail('Cancelled.')
    return HttpResponseRedirect(request.path)

<commit_msg>Drop chunk size down to 10000<commit_after>import logging
from datetime import datetime

from django.contrib import admin
from django.http import HttpResponseRedirect
from django.shortcuts import render

from pyelasticsearch.exceptions import (
    ConnectionError, ElasticHttpNotFoundError, Timeout)

from fjord.search.index import (
    chunked, get_index, get_index_stats, get_indexes, get_indexable,
    recreate_index, create_batch_id)
from fjord.search.models import Record
from fjord.search.tasks import index_chunk_task


log = logging.getLogger('i.search')


CHUNK_SIZE = 10000


def handle_reset(request):
    """Mark outstanding Records as failed.

    Why? You'd want to reset the system if it gets itself wedged
    thinking there are outstanding tasks, but there aren't. This
    lets you fix that.

    """
    for rec in Record.outstanding():
        rec.mark_fail('Cancelled.')
    return HttpResponseRedirect(request.path)


{'fraction_non_alphanumeric': 0.06638115631691649, 'fraction_numerical': 0.008029978586723769, 'mean_word_length': 4.433139534883721, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4853546', 'n_tokens_mistral': 549, 'n_tokens_neox': 521, 'n_words': 175}