<commit_before>from django.core.management.base import NoArgsCommand
from django.template import loader, Context
from haystack.constants import DEFAULT_OPERATOR
from haystack.fields import *


class Command(NoArgsCommand):
    help = "Generates a Solr schema that reflects the indexes."
    
    def handle_noargs(self, **options):
        """Generates a Solr schema that reflects the indexes."""
        # Cause the default site to load.
        from django.conf import settings
        __import__(settings.ROOT_URLCONF)
        from haystack.sites import site
        
        content_field_name = ''
        fields = []
        default_operator = getattr(settings, 'HAYSTACK_DEFAULT_OPERATOR', DEFAULT_OPERATOR)
        
        for model, index in site.get_indexes().items():
            for field_name, field_object in index.fields.items():
                field_data = {
                    'field_name': field_name,
                    'type': 'text',
                    'indexed': 'true',
                    'multi_valued': 'false',
                }
                
                # Nasty but...
                if isinstance(field_object, ContentField):
                    content_field_name = field_name
                elif isinstance(field_object, StoredField):
                    field_data['indexed'] = 'false'
                elif isinstance(field_object, DateField) or isinstance(field_object, DateTimeField):
                    field_data['type'] = 'date'
                elif isinstance(field_object, IntegerField):
                    field_data['type'] = 'slong'
                elif isinstance(field_object, FloatField):
                    field_data['type'] = 'sfloat'
                elif isinstance(field_object, BooleanField):
                    field_data['type'] = 'boolean'
                elif isinstance(field_object, MultiValueField):
                    field_data['multi_valued'] = 'true'
            
                fields.append(field_data)<commit_msg>No more duplicates from the "build_solr_schema" management command.<commit_after>from django.core.management.base import NoArgsCommand
from django.template import loader, Context
from haystack.constants import DEFAULT_OPERATOR
from haystack.fields import *
try:
    set
except NameError:
    from sets import Set as set


class Command(NoArgsCommand):
    help = "Generates a Solr schema that reflects the indexes."
    
    def handle_noargs(self, **options):
        """Generates a Solr schema that reflects the indexes."""
        # Cause the default site to load.
        from django.conf import settings
        __import__(settings.ROOT_URLCONF)
        from haystack.sites import site
        
        content_field_name = ''
        fields = []
        field_names = set()
        default_operator = getattr(settings, 'HAYSTACK_DEFAULT_OPERATOR', DEFAULT_OPERATOR)
        
        for model, index in site.get_indexes().items():
            for field_name, field_object in index.fields.items():
                if field_name in field_names:
                    # We've already got this field in the list. Skip.
                    continue
                
                field_names.add(field_name)
                
                field_data = {
                    'field_name': field_name,
                    'type': 'text',
                    'indexed': 'true',
                    'multi_valued': 'false',
                }
                
                # Nasty but...
                if isinstance(field_object, ContentField):
                    content_field_name = field_name
                elif isinstance(field_object, StoredField):
                    field_data['indexed'] = 'false'
                elif isinstance(field_object, DateField) or isinstance(field_object, DateTimeField):
                    field_data['type'] = 'date'
                elif isinstance(field_object, IntegerField):
                    field_data['type'] = 'slong'
                elif isinstance(field_object, FloatField):
                    field_data['type'] = 'sfloat'
                elif isinstance(field_object, BooleanField):
                    field_data['type'] = 'boolean'
                elif isinstance(field_object, MultiValueField):
                    field_data['multi_valued'] = 'true'
            
                fields.append(field_data)
{'fraction_non_alphanumeric': 0.08203483043079743, 'fraction_numerical': 0.0, 'mean_word_length': 1.7927063339731286, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6098834', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1033, 'n_words': 271}