<commit_before>"""
Models meta magic here
"""
from pso.fields import BaseField
from pso.query import BaseQuerySet


class ModelMetaClass(type):
    """
    Magic with fields
    """

    def __new__(meta, name, bases, attr_dict):
        attr_dict['_fields'] = []
        attr_dict['_stored_fields'] = []
        queryset_class = attr_dict.pop('__queryset_class__', BaseQuerySet)
        fields = []
        stored_fields = []

        for name, attr in attr_dict.items():
            if isinstance(attr, list) and len(attr) == 1 \
               and isinstance(attr[0], BaseField):
                # this allows to define models like
                # tage = list(TextField())
                attr = attr[0]
                attr_dict[name] = attr  # replace attr in object
                attr.multi_valued = True

            if isinstance(attr, BaseField):

                if not attr.name:
                    attr.name = name
                fields.append(name)
                if attr.store:
                    stored_fields.append(name)

            attr_dict['_fields'] = fields
            attr_dict['_stored_fields'] = stored_fields

        cls = super().__new__(meta, name, bases, attr_dict)

        # Bind QuerySet  # TODO check this
        setattr(cls, 'objects', queryset_class(cls))
        return cls


class BaseModel(metaclass=ModelMetaClass):
    """Base class for EngineSpecific models"""
    _fields = []
    _stored_fields = []
    _cache = {}

    def __init__(self, **kwargs):
        for key, arg in kwargs.items():
            if key in self._fields:
                setattr(self, key, arg)

    @classmethod
    def get_fields(cls):
        print(cls, cls._fields)
        return (getattr(cls, name) for name in cls._fields)

    def __iter__(self):
        """Only filed with data to emulate dict behavior"""
        for name in self._fields:
            yield name, self[name]

    def __repr__(self):
        return "<Model: {0.__class__.__name__}>".format(self)

    def __getitem__(self, key):
        return self._cache[key]

    def to_index(self):
        # return self._cache
        arr = {}
        for name in self._fields:
            field = getattr(self.__class__, name)
            if name == '_version_':
                continue
            if field.multi_valued:
                arr[name] = [field.to_index(val) for val in self._cache.get(name, [])]
            else:
                arr[name] = field.to_index(self._cache.get(name))
            return arr
      # def get_object<commit_msg>Fix wrong _cache<commit_after>"""
Models meta magic here
"""
from pso.fields import BaseField
from pso.query import BaseQuerySet


class ModelMetaClass(type):
    """
    Magic with fields
    """

    def __new__(meta, name, bases, attr_dict):
        attr_dict['_fields'] = []
        attr_dict['_stored_fields'] = []
        queryset_class = attr_dict.pop('__queryset_class__', BaseQuerySet)
        fields = []
        stored_fields = []

        for name, attr in attr_dict.items():
            if isinstance(attr, list) and len(attr) == 1 \
               and isinstance(attr[0], BaseField):
                # this allows to define models like
                # tage = list(TextField())
                attr = attr[0]
                attr_dict[name] = attr  # replace attr in object
                attr.multi_valued = True

            if isinstance(attr, BaseField):

                if not attr.name:
                    attr.name = name
                fields.append(name)
                if attr.store:
                    stored_fields.append(name)

        attr_dict['_fields'] = fields
        attr_dict['_stored_fields'] = stored_fields
        attr_dict['_cache'] = {}

        cls = super().__new__(meta, name, bases, attr_dict)

        # Bind QuerySet  # TODO check this
        setattr(cls, 'objects', queryset_class(cls))
        return cls


class BaseModel(metaclass=ModelMetaClass):
    """Base class for EngineSpecific models"""

    def __init__(self, **kwargs):
        self._cache = {}
        for key, arg in kwargs.items():
            if key in self._fields:
                self._cache[key] = arg

    @classmethod
    def get_fields(cls):
        print(cls, cls._fields)
        return (getattr(cls, name) for name in cls._fields)

    def __iter__(self):
        """Only filed with data to emulate dict behavior"""
        for name in self._fields:
            yield name, self[name]

    def __repr__(self):
        return "<Model: {0.__class__.__name__}>".format(self)

    def __getitem__(self, key):
        return self._cache[key]

    def to_index(self):
        # return self._cache
        arr = {}
        for name in self._fields:
            field = getattr(self.__class__, name)
            if name == '_version_':
                continue
            if field.multi_valued:
                arr[name] = [field.to_index(val) for val in self._cache.get(name, [])]
            else:
                arr[name] = field.to_index(self._cache.get(name))
        return arr
      # def get_object
{'fraction_non_alphanumeric': 0.10600985221674877, 'fraction_numerical': 0.0015763546798029558, 'mean_word_length': 1.8808172531214529, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3655512', 'n_tokens_mistral': 1470, 'n_tokens_neox': 1415, 'n_words': 375}