<commit_before>from six import iteritems

from django.template import Context, loader
from django.template.defaultfilters import striptags
from elasticsearch_dsl.analysis import Analyzer


class AbstractField(object):
    '''
    Represents an elasticsearch index field and values from given objects.
    Currently does not support binary fields, but those can be created by manually providing a dictionary.

    Values are extracted using the `model_attr` or `eval_as` attribute.
    '''
    common_fields = ['index_name', 'store', 'index', 'boost', 'null_value', 'copy_to']
    @property
    def fields(self):
        try:
            return self.fields
        except:
            raise NotImplementedError('Allowed fields are not defined.')

    @property
    def coretype(self):
        try:<commit_msg>Add 'type' and 'fields' to AbstractField.common_fields<commit_after>from six import iteritems

from django.template import Context, loader
from django.template.defaultfilters import striptags
from elasticsearch_dsl.analysis import Analyzer


class AbstractField(object):
    '''
    Represents an elasticsearch index field and values from given objects.
    Currently does not support binary fields, but those can be created by manually providing a dictionary.

    Values are extracted using the `model_attr` or `eval_as` attribute.
    '''
    common_fields = ['index_name', 'store', 'index', 'boost', 'null_value', 'copy_to', 'type', 'fields']
    @property
    def fields(self):
        try:
            return self.fields
        except:
            raise NotImplementedError('Allowed fields are not defined.')

    @property
    def coretype(self):
        try:
{'fraction_non_alphanumeric': 0.08059701492537313, 'fraction_numerical': 0.0, 'mean_word_length': 3.493297587131367, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3789828', 'n_tokens_mistral': 435, 'n_tokens_neox': 426, 'n_words': 162}