<commit_before>import logging, sys
from flask import flash
import filters
from ..._compat import as_unicode
from ..base import BaseInterface
from ..filters import Filters
from mongoengine.fields import StringField, IntField, BooleanField, FloatField, DateTimeField, ReferenceField

log = logging.getLogger(__name__)


def _include_filters(obj):
    for key in filters.__all__:
        if not hasattr(obj, key):
            setattr(obj, key, getattr(filters, key))


class MongoEngineInterface(BaseInterface):

    filter_converter_class = filters.MongoEngineFilterConverter

    def __init__(self, obj, session=None):
        self.session = session
        _include_filters(self)
        super(MongoEngineInterface, self).__init__(obj)

    @property
    def model_name(self):
        """
            Returns the models class name
            useful for auto title on views
        """
        return self.obj.__name__

    def query(self, filters=None, order_column='', order_direction='',
              page=None, page_size=None):

        if filters:
            objs = filters.apply_all(self.obj.objects)
        else:
            objs = self.obj.objects
        count = len(objs)
        start, stop = 0, count
        if page:
            start = page * page_size
        if page_size:
            stop = start + page_size
        if order_column != '':
            if order_direction == 'asc':
                objs = objs.order_by('-{0}'.format(order_column))
            else:
                objs = objs.order_by('+{0}'.format(order_column))
        return count, objs[start:stop]

    def is_string(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], StringField)
        except:
            return False

    def is_integer(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], IntField)
        except:
            return False

    def is_float(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], FloatField)
        except:
            return False

    def is_boolean(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], BooleanField)
        except:
            return False

    def is_datetime(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], DateTimeField)
        except:
            return False

    def is_relation(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], ReferenceField)
        except:
            return False

    def is_relation_many_to_one(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], ReferenceField)
        except:
            return False

    def is_relation_many_to_many(self, col_name):
        return False

    def is_relation_one_to_one(self, col_name):
        return False

    def is_relation_one_to_many(self, col_name):
        return False

    def is_nullable(self, col_name):
        return not self.obj._fields[col_name].required

    def is_unique(self, col_name):
        return self.obj._fields[col_name].unique

    def is_pk(self, col_name):
        return col_name == 'id'

    def add(self, item):
        try:
            item.save()
            flash(as_unicode(self.add_row_message), 'success')
            return True
        except Exception as e:
            flash(as_unicode(self.general_error_message + ' ' + str(sys.exc_info()[0])), 'danger')
            log.exception("Add record error: {0}".format(str(e)))
            return False

    def edit(self, item):
        try:
            item.save()
            flash(as_unicode(self.edit_row_message), 'success')
            return True
        except Exception as e:
            flash(as_unicode(self.general_error_message + ' ' + str(sys.exc_info()[0])), 'danger')
            log.exception("Edit record error: {0}".format(str(e)))
            return False

    def delete(self, item):
        try:
            item.delete()
            flash(as_unicode(self.delete_row_message), 'success')
            return True
        except Exception as e:
            flash(as_unicode(self.general_error_message + ' ' + str(sys.exc_info()[0])), 'danger')
            log.exception("Delete record error: {0}".format(str(e)))
            return False

    def get_columns_list(self):
        return self.obj._fields.keys()

    def get_search_columns_list(self):
        ret_lst = list()
        for col_name in self.get_columns_list():
            for conversion in self.filter_converter_class.conversion_table:
                if getattr(self, conversion[0])(col_name):
                    ret_lst.append(col_name)
        return ret_lst

    def get_user_columns_list(self):
        """
            Returns all model's columns except pk or fk
        """
        ret_lst = list()
        for col_name in self.get_columns_list():
            if (not self.is_pk(col_name)) and (not self.is_fk(col_name)):
                ret_lst.append(col_name)
        return ret_lst

    def get_order_columns_list(self, list_columns=None):
        return self.obj._fields.keys()

    def get_related_model(self, col_name):
        return self.obj._fields[col_name].document_type

    def get_related_interface(self, col_name):<commit_msg>MongoEngine support for many to many with ListField and ReferenceField<commit_after>import logging, sys
from flask import flash
import filters
from ..._compat import as_unicode
from ..base import BaseInterface
from mongoengine.fields import StringField, IntField, BooleanField, FloatField, \
    DateTimeField, ReferenceField, ListField, Document

log = logging.getLogger(__name__)


def _include_filters(obj):
    for key in filters.__all__:
        if not hasattr(obj, key):
            setattr(obj, key, getattr(filters, key))


class MongoEngineInterface(BaseInterface):

    filter_converter_class = filters.MongoEngineFilterConverter

    def __init__(self, obj, session=None):
        self.session = session
        _include_filters(self)
        super(MongoEngineInterface, self).__init__(obj)

    @property
    def model_name(self):
        """
            Returns the models class name
            useful for auto title on views
        """
        return self.obj.__name__

    def query(self, filters=None, order_column='', order_direction='',
              page=None, page_size=None):

        if filters:
            objs = filters.apply_all(self.obj.objects)
        else:
            objs = self.obj.objects
        count = len(objs)
        start, stop = 0, count
        if page:
            start = page * page_size
        if page_size:
            stop = start + page_size
        if order_column != '':
            if order_direction == 'asc':
                objs = objs.order_by('-{0}'.format(order_column))
            else:
                objs = objs.order_by('+{0}'.format(order_column))
        return count, objs[start:stop]

    def is_string(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], StringField)
        except:
            return False

    def is_integer(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], IntField)
        except:
            return False

    def is_float(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], FloatField)
        except:
            return False

    def is_boolean(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], BooleanField)
        except:
            return False

    def is_datetime(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], DateTimeField)
        except:
            return False

    def is_relation(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], ReferenceField) or \
                    isinstance(self.obj._fields[col_name], ListField)
        except:
            return False

    def is_relation_many_to_one(self, col_name):
        try:
            return isinstance(self.obj._fields[col_name], ReferenceField)
        except:
            return False

    def is_relation_many_to_many(self, col_name):
        try:
            field = self.obj._fields[col_name]
            return isinstance(field, ListField) and isinstance(field.field, ReferenceField)
        except:
            return False


    def is_relation_one_to_one(self, col_name):
        return False

    def is_relation_one_to_many(self, col_name):
        return False

    def is_nullable(self, col_name):
        return not self.obj._fields[col_name].required

    def is_unique(self, col_name):
        return self.obj._fields[col_name].unique

    def is_pk(self, col_name):
        return col_name == 'id'

    def add(self, item):
        try:
            item.save()
            flash(as_unicode(self.add_row_message), 'success')
            return True
        except Exception as e:
            flash(as_unicode(self.general_error_message + ' ' + str(sys.exc_info()[0])), 'danger')
            log.exception("Add record error: {0}".format(str(e)))
            return False

    def edit(self, item):
        try:
            item.save()
            flash(as_unicode(self.edit_row_message), 'success')
            return True
        except Exception as e:
            flash(as_unicode(self.general_error_message + ' ' + str(sys.exc_info()[0])), 'danger')
            log.exception("Edit record error: {0}".format(str(e)))
            return False

    def delete(self, item):
        try:
            item.delete()
            flash(as_unicode(self.delete_row_message), 'success')
            return True
        except Exception as e:
            flash(as_unicode(self.general_error_message + ' ' + str(sys.exc_info()[0])), 'danger')
            log.exception("Delete record error: {0}".format(str(e)))
            return False

    def get_columns_list(self):
        return self.obj._fields.keys()

    def get_search_columns_list(self):
        ret_lst = list()
        for col_name in self.get_columns_list():
            for conversion in self.filter_converter_class.conversion_table:
                if getattr(self, conversion[0])(col_name):
                    ret_lst.append(col_name)
        return ret_lst

    def get_user_columns_list(self):
        """
            Returns all model's columns except pk or fk
        """
        ret_lst = list()
        for col_name in self.get_columns_list():
            if (not self.is_pk(col_name)) and (not self.is_fk(col_name)):
                ret_lst.append(col_name)
        return ret_lst

    def get_order_columns_list(self, list_columns=None):
        return self.obj._fields.keys()

    def get_related_model(self, col_name):
        field = self.obj._fields[col_name]
        if isinstance(field, ListField):
            return field.field.document_type
        else:
            return field.document_type

    def get_related_interface(self, col_name):
{'fraction_non_alphanumeric': 0.09551886792452831, 'fraction_numerical': 0.00181422351233672, 'mean_word_length': 2.204010462074978, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2883352', 'n_tokens_mistral': 3206, 'n_tokens_neox': 3108, 'n_words': 719}