<commit_before>"""rw.db provides a simple database ORM for mongodb

Example::

    from rw.www import RequestHandler, get, post
    import motor
    import rw.db
    from tornado import gen

    connection = motor.MotorConnection(max_pool_size=10, max_concurrent=20).open_sync()
    rw.db.db = connection.test


    class User(rw.db.Entity):
        name = rw.db.Field(unicode)
        email = rw.db.Field(unicode)


    class Main(RequestHandler):
        @gen.engine
        @get('/')
        def index(self):
            self['users'] = yield gen.Task(User.query.filter_by(name='2').all)
            self.finish(template='index.html')

"""
import numbers
import abc

from copy import copy
from tornado.concurrent import return_future

db = None

__all__ = ['Entity']


class Cursor(object):
    def __init__(self, query, user_callback=None):
        self.user_callback = user_callback
        self.col_cls = query.col_cls
        col = getattr(db, query.col_cls._name)
        print 'query', col, query._filters
        self.db_cursor = col.find(query._filters, sort=query._sort, limit=query._limit)
        self.db_cursor.to_list(callback=self.on_response)

    def on_response(self, response, error):
        assert not error
        self.user_callback([self.col_cls(**e) for e in response])


class Query(object):
    def __init__(self, col, filters=None, sort=None, limit=0, start=0):
        self.col_cls = col
        self._sort = sort
        if db:
            self.col = getattr(db, col._name)
        self._filters = filters if filters else {}
        self._limit = limit
        self._start = start

    def __getitem__(self, slice):
        if isinstance(slice, numbers.Number):
            return Query(self.col_cls, self._filters, self.sort,
                         limit=1, start=slice)
        elif slice.step is None \
          and isinstance(slice.start, numbers.Number)\
          and isinstance(slice.stop, numbers.Number)\
          and slice.stop > slice.start:
            return Query(self.col_cls, self._filters, self._sort,
                         limit=slice.stop - slice.start, start=slice.start)
        else:
            raise AttributeError('Slice indecies must be integers, step (= {}) must not be set'
                                 ' and start (= {}) must be higher than stop (= {})'.format(
                                 slice.step, repr(slice.start), repr(slice.stop)
            ))

    def find(self, *args, **kwargs):
        filters = copy(self._filters)
        filters.update(kwargs)
        if args:
            assert len(args) == 1
            filters.update(args[0])
        return Query(self.col_cls, filters, self._sort, self._limit)

    def sort(self, sort):
        return Query(self.col_cls, self._filters, sort, self._limit)

    @return_future
    def to_list(self, callback):
        Cursor(self, callback)

    @return_future
    def first(self, callback):
        self._limit = 1
        Cursor(self, callback)

    def get(self, value, user_callback):
        def callback(elements, error):
            if elements:
                user_callback(elements[0])
            else:
                user_callback(None)
        self.col.find({'_id': value}, callback=callback)


class Field(property):
    def __init__(self, type, _id=False, default=None):
        # print 'init property', self, type
        super(Field, self).__init__(self.get_value, self.set_value)
        self.name = None
        self.type = type
        self.default = default
        self.is_id = _id

    def get_value(self, entity):
        if self.is_id:
            return entity['_id']
        else:
            # try:
                return self.type(entity[self.name])
            # except:
            #     return self.default

    def set_value(self, entity, value):
        if self.is_id:
            entity['_id'] = value
        else:
            entity[self.name] = value

    def __repr__(self):
        return '<Field %i>' % self.name


# TODO
class List(Field):
    pass


# TODO
class Reference(Field):
    pass


class DocumentMeta(type):
    def __new__(mcs, name, bases, dct):
        ret = type.__new__(mcs, name, bases, dct)

        ret._id_name = '_id'
        for key, value in dct.items():
            if isinstance(value, Field):
                field = getattr(ret, key)
                field.name = key
                if field.is_id:
                    if ret._id_name == '_id':
                        ret._id_name = key
                    else:
                        raise AttributeError('Two fields with _id=True %s, %s'
                                             % (ret._id_name, value))

        if bases != (dict,):
            ret._name = name.lower()
            # ret.query = Query(ret)

        return ret


class Document(dict):<commit_msg>remove is_id from rw.db<commit_after>"""rw.db provides a simple database ORM for mongodb

Example::

    from rw.www import RequestHandler, get, post
    import motor
    import rw.db
    from tornado import gen

    connection = motor.MotorConnection(max_pool_size=10, max_concurrent=20).open_sync()
    rw.db.db = connection.test


    class User(rw.db.Entity):
        name = rw.db.Field(unicode)
        email = rw.db.Field(unicode)


    class Main(RequestHandler):
        @gen.engine
        @get('/')
        def index(self):
            self['users'] = yield gen.Task(User.query.filter_by(name='2').all)
            self.finish(template='index.html')

"""
import numbers
import abc

from copy import copy
from tornado.concurrent import return_future

db = None

__all__ = ['Entity']


class Cursor(object):
    def __init__(self, query, user_callback=None):
        self.user_callback = user_callback
        self.col_cls = query.col_cls
        col = getattr(db, query.col_cls._name)
        print 'query', col, query._filters
        self.db_cursor = col.find(query._filters, sort=query._sort, limit=query._limit)
        self.db_cursor.to_list(callback=self.on_response)

    def on_response(self, response, error):
        assert not error
        self.user_callback([self.col_cls(**e) for e in response])


class Query(object):
    def __init__(self, col, filters=None, sort=None, limit=0, start=0):
        self.col_cls = col
        self._sort = sort
        if db:
            self.col = getattr(db, col._name)
        self._filters = filters if filters else {}
        self._limit = limit
        self._start = start

    def __getitem__(self, slice):
        if isinstance(slice, numbers.Number):
            return Query(self.col_cls, self._filters, self.sort,
                         limit=1, start=slice)
        elif slice.step is None \
          and isinstance(slice.start, numbers.Number)\
          and isinstance(slice.stop, numbers.Number)\
          and slice.stop > slice.start:
            return Query(self.col_cls, self._filters, self._sort,
                         limit=slice.stop - slice.start, start=slice.start)
        else:
            raise AttributeError('Slice indecies must be integers, step (= {}) must not be set'
                                 ' and start (= {}) must be higher than stop (= {})'.format(
                                 slice.step, repr(slice.start), repr(slice.stop)
            ))

    def find(self, *args, **kwargs):
        filters = copy(self._filters)
        filters.update(kwargs)
        if args:
            assert len(args) == 1
            filters.update(args[0])
        return Query(self.col_cls, filters, self._sort, self._limit)

    def sort(self, sort):
        return Query(self.col_cls, self._filters, sort, self._limit)

    @return_future
    def to_list(self, callback):
        Cursor(self, callback)

    @return_future
    def first(self, callback):
        self._limit = 1
        Cursor(self, callback)

    def get(self, value, user_callback):
        def callback(elements, error):
            if elements:
                user_callback(elements[0])
            else:
                user_callback(None)
        self.col.find({'_id': value}, callback=callback)


class NoDefaultValue(object):
    pass


class Field(property):
    def __init__(self, type, default=NoDefaultValue):
        # print 'init property', self, type
        super(Field, self).__init__(self.get_value, self.set_value)
        self.name = None
        self.type = type
        self.default = default

    def get_value(self, entity):
        if self.name in entity:
            value = entity[self.name]
        elif self.default is not NoDefaultValue:
            value = self.default
        else:
            raise ValueError('Value not found')
        return self.type(value)

    def set_value(self, entity, value):
        entity[self.name] = value

    def __repr__(self):
        return '<Field %i>' % self.name


# TODO
class List(Field):
    pass


# TODO
class Reference(Field):
    pass


class DocumentMeta(type):
    def __new__(mcs, name, bases, dct):
        ret = type.__new__(mcs, name, bases, dct)

        ret._id_name = '_id'
        for key, value in dct.items():
            if isinstance(value, Field):
                field = getattr(ret, key)
                field.name = key

        if bases != (dict,):
            ret._name = name.lower()
            # ret.query = Query(ret)

        return ret


class Document(dict):
{'fraction_non_alphanumeric': 0.10966085374986627, 'fraction_numerical': 0.002567668770728576, 'mean_word_length': 2.0213316095669036, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '151030', 'n_tokens_mistral': 2763, 'n_tokens_neox': 2662, 'n_words': 685}