<commit_before>import persistent
import ZODB
from ZEO import ClientStorage
import zc.zlibstorage
import transaction
import logging
from pickle import loads
from repoze.catalog.query import Contains
import time
from zerodb.catalog import Catalog, CatalogTextIndex, CatalogFieldIndex
from zerodb.trees import family32
import random


logging.basicConfig(level=logging.DEBUG)


class DebugStorage(zc.zlibstorage.ZlibStorage):
    def load(self, oid, version=''):
        data, serial = self.base.load(oid, version)
        obj = self._untransform(data)
        print oid.encode("hex"), loads(obj), len(data), "->", len(obj)
        return obj, serial


class Page(persistent.Persistent):

    def __init__(self, title="", text=""):
        self.title = title
        self.text = text


class Salary(persistent.Persistent):
    def __init__(self, name="", surname="", salary=0):
        self.name = name
        self.surname = surname
        self.salary = salary


def create_objects(root, count=20000):
    def p_record_and_index(i, **kw):
        obj = Page(**kw)
        root["pages"][i] = obj
        root["pages_catalog"].index_doc(i, obj)

    def s_record_and_index(i, **kw):
        obj = Salary(**kw)
        root["salaries"][i] = obj
        root["salaries_catalog"].index_doc(i, obj)

    for i in range(count / 2):
        p_record_and_index(i, title="hello %s" % i, text="lorem ipsum dolor sit amet" * 50)
    for i in range(count / 2, count / 2 + 10):
        p_record_and_index(i, title="hello %s" % i, text="this is something we're looking for" * 50)
    for i in range(count / 2 + 10, count):
        p_record_and_index(i, title="hello %s" % i, text="lorem ipsum dolor sit amet" * 50)

    for i in range(count):
        s_record_and_index(i, name="John" + str(i), surname="Smith" + str(i), salary=random.randrange(50000, 200000))


def make_zodb():
    db = ZODB.DB(DebugStorage(ClientStorage.ClientStorage("/tmp/zeosocket")))
    conn = db.open()
    root = conn.root()

    if 'pages_catalog' not in root.keys():
        catalog = Catalog()
        catalog["title"] = CatalogFieldIndex("title")
        catalog["text"] = CatalogTextIndex("text")
        root["pages_catalog"] = catalog

    if 'salaries_catalog' not in root.keys():
        catalog = Catalog()
        catalog["name"] = CatalogFieldIndex("name")
        catalog["surname"] = CatalogFieldIndex("surname")
        catalog["salary"] = CatalogFieldIndex("salary")
        root["salaries_catalog"] = catalog

    if 'pages' not in root.keys():
        root["pages"] = family32.IO.BTree()
        root["salaries"] = family32.IO.BTree()
        create_objects(root)

    return root


def test_query_1(root):
    t1 = time.time()
    ids = root['pages_catalog'].query(Contains("text", "something"))
    print "===", time.time() - t1
    return [root["pages"][i] for i in ids[1]]


def test_query_2(root):
    from repoze.catalog.query import InRange
    # return root["salaries_catalog"].query(Gt("salary", 130000) & Lt("salary", 130050))
    # logical multiplication of two conditions is totally ineffective, apparently
    # E.g. handled totally out of db (so, that needs some work, obviously!)
    return root["salaries_catalog"].query(InRange("salary", 130000, 130050))


if __name__ == "__main__":
    root = make_zodb()<commit_msg>Sort index + limits (left after tests)<commit_after>import persistent
import ZODB
from ZEO import ClientStorage
import zc.zlibstorage
import transaction
import logging
from pickle import loads
from repoze.catalog.query import Contains
import time
from zerodb.catalog import Catalog, CatalogTextIndex, CatalogFieldIndex
from zerodb.trees import family32
import random


logging.basicConfig(level=logging.DEBUG)


class DebugStorage(zc.zlibstorage.ZlibStorage):
    def load(self, oid, version=''):
        data, serial = self.base.load(oid, version)
        obj = self._untransform(data)
        print oid.encode("hex"), loads(obj), len(data), "->", len(obj)
        return obj, serial


class Page(persistent.Persistent):

    def __init__(self, title="", text=""):
        self.title = title
        self.text = text


class Salary(persistent.Persistent):
    def __init__(self, name="", surname="", salary=0):
        self.name = name
        self.surname = surname
        self.salary = salary


def create_objects(root, count=20000):
    def p_record_and_index(i, **kw):
        obj = Page(**kw)
        root["pages"][i] = obj
        root["pages_catalog"].index_doc(i, obj)

    def s_record_and_index(i, **kw):
        obj = Salary(**kw)
        root["salaries"][i] = obj
        root["salaries_catalog"].index_doc(i, obj)

    for i in range(count / 2):
        p_record_and_index(i, title="hello %s" % i, text="lorem ipsum dolor sit amet" * 50)
    for i in range(count / 2, count / 2 + 10):
        p_record_and_index(i, title="hello %s" % i, text="this is something we're looking for" * 50)
    for i in range(count / 2 + 10, count):
        p_record_and_index(i, title="hello %s" % i, text="lorem ipsum dolor sit amet" * 50)

    for i in range(count):
        s_record_and_index(i, name="John" + str(i), surname="Smith" + str(i), salary=random.randrange(50000, 200000))


def make_zodb():
    db = ZODB.DB(DebugStorage(ClientStorage.ClientStorage("/tmp/zeosocket")))
    conn = db.open()
    root = conn.root()

    if 'pages_catalog' not in root.keys():
        catalog = Catalog()
        catalog["title"] = CatalogFieldIndex("title")
        catalog["text"] = CatalogTextIndex("text")
        root["pages_catalog"] = catalog

    if 'salaries_catalog' not in root.keys():
        catalog = Catalog()
        catalog["name"] = CatalogFieldIndex("name")
        catalog["surname"] = CatalogFieldIndex("surname")
        catalog["salary"] = CatalogFieldIndex("salary")
        root["salaries_catalog"] = catalog

    if 'pages' not in root.keys():
        root["pages"] = family32.IO.BTree()
        root["salaries"] = family32.IO.BTree()
        create_objects(root)

    return root


def test_query_1(root):
    t1 = time.time()
    ids = root['pages_catalog'].query(Contains("text", "something"), sort_index="title", limit=2)
    print "===", time.time() - t1
    return [root["pages"][i] for i in ids[1]]


def test_query_2(root):
    from repoze.catalog.query import InRange
    # return root["salaries_catalog"].query(Gt("salary", 130000) & Lt("salary", 130050))
    # logical multiplication of two conditions is totally ineffective, apparently
    # E.g. handled totally out of db (so, that needs some work, obviously!)
    return root["salaries_catalog"].query(InRange("salary", 130000, 130050), sort_index="salary", limit=2)


if __name__ == "__main__":
    root = make_zodb()
{'fraction_non_alphanumeric': 0.13930274135876042, 'fraction_numerical': 0.019964243146603097, 'mean_word_length': 3.466400532268796, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 4, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2964175', 'n_tokens_mistral': 2292, 'n_tokens_neox': 2108, 'n_words': 544}