<commit_before>from datetime import datetime

from bungiesearch import Bungiesearch
from bungiesearch.utils import update_index
from django.core.management import call_command
from django.test import TestCase
import pytz
from six import iteritems

from core.models import Article, User, Unmanaged, NoUpdatedField, ManangedButEmpty
from core.search_indices import ArticleIndex, UserIndex


class CoreTestCase(TestCase):
    @classmethod
    def setUpClass(cls):
        # Let's start by creating the index and mapping.
        # If we create an object before the index, the index
        # will be created automatically, and we want to test the command.
        call_command('search_index', action='create')

        art_1 = {'title': 'Title one',
                 'description': 'Description of article 1.',
                 'text_field': '',
                 'link': 'http://example.com/article_1',
                 'published': pytz.UTC.localize(datetime(year=2020, month=9, day=15)),
                 'updated': pytz.UTC.localize(datetime(year=2014, month=9, day=10)),
                 'tweet_count': 20,
                 'source_hash': 159159159159,
                 'missing_data': '',
                 'positive_feedback': 50,
                 'negative_feedback': 5,
                 }

        user_1 = {'user_id': 'bungie1',
                  'description': 'Description of user 1',
                  'created': pytz.UTC.localize(datetime(year=2015, month=1, day=1)),
                  'updated': pytz.UTC.localize(datetime(year=2015, month=6, day=1)),
                 }

        Article.objects.create(**art_1)
        User.objects.create(**user_1)

        art_2 = dict((k, v) for k, v in iteritems(art_1))
        art_2['link'] += '/page2'
        art_2['title'] = 'Title two'
        art_2['description'] = 'This is a second article.'
        art_2['text_field'] = None
        art_2['published'] = pytz.UTC.localize(datetime(year=2010, month=9, day=15))

        user_2 = dict((k, v) for k, v in iteritems(user_1))
        user_2['user_id'] = 'bungie2'
        user_2['description'] = 'This is the second user'
        user_2['created'] = pytz.UTC.localize(datetime(year=2010, month=9, day=15))

        Article.objects.create(**art_2)
        User.objects.create(**user_2)
        NoUpdatedField.objects.create(title='My title', description='This is a short description.')

        call_command('rebuild_index', interactive=False)

    @classmethod<commit_msg>Added guilty-as-charged flag to rebuild_index.<commit_after>from datetime import datetime

from bungiesearch import Bungiesearch
from bungiesearch.utils import update_index
from django.core.management import call_command
from django.test import TestCase
import pytz
from six import iteritems

from core.models import Article, User, Unmanaged, NoUpdatedField, ManangedButEmpty
from core.search_indices import ArticleIndex, UserIndex


class CoreTestCase(TestCase):
    @classmethod
    def setUpClass(cls):
        # Let's start by creating the index and mapping.
        # If we create an object before the index, the index
        # will be created automatically, and we want to test the command.
        call_command('search_index', action='create')

        art_1 = {'title': 'Title one',
                 'description': 'Description of article 1.',
                 'text_field': '',
                 'link': 'http://example.com/article_1',
                 'published': pytz.UTC.localize(datetime(year=2020, month=9, day=15)),
                 'updated': pytz.UTC.localize(datetime(year=2014, month=9, day=10)),
                 'tweet_count': 20,
                 'source_hash': 159159159159,
                 'missing_data': '',
                 'positive_feedback': 50,
                 'negative_feedback': 5,
                 }

        user_1 = {'user_id': 'bungie1',
                  'description': 'Description of user 1',
                  'created': pytz.UTC.localize(datetime(year=2015, month=1, day=1)),
                  'updated': pytz.UTC.localize(datetime(year=2015, month=6, day=1)),
                 }

        Article.objects.create(**art_1)
        User.objects.create(**user_1)

        art_2 = dict((k, v) for k, v in iteritems(art_1))
        art_2['link'] += '/page2'
        art_2['title'] = 'Title two'
        art_2['description'] = 'This is a second article.'
        art_2['text_field'] = None
        art_2['published'] = pytz.UTC.localize(datetime(year=2010, month=9, day=15))

        user_2 = dict((k, v) for k, v in iteritems(user_1))
        user_2['user_id'] = 'bungie2'
        user_2['description'] = 'This is the second user'
        user_2['created'] = pytz.UTC.localize(datetime(year=2010, month=9, day=15))

        Article.objects.create(**art_2)
        User.objects.create(**user_2)
        NoUpdatedField.objects.create(title='My title', description='This is a short description.')

        call_command('rebuild_index', interactive=False, confirmed='guilty-as-charged')

    @classmethod
{'fraction_non_alphanumeric': 0.12260765550239235, 'fraction_numerical': 0.03229665071770335, 'mean_word_length': 2.5912670007158196, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3605699', 'n_tokens_mistral': 1565, 'n_tokens_neox': 1449, 'n_words': 362}