<commit_before>import time
import string
import logging
from collections import namedtuple, OrderedDict
from contextlib import contextmanager

import xapian
import nilsimsa

from operator import itemgetter
from functools import partial
from json import loads

from xapian import Query, QueryParser, DocNotFoundError

from . import snowball
from .elements import Schema
from .exc import ValidationError, PrefixError
from .tools import LRUDict, lazy_property


RETRY_LIMIT = 5
RETRY_BACKOFF_FACTOR = 0.2

logger = logging.getLogger(__name__)

XAPIAN_VERSION = namedtuple('XAPIAN_VERSION', ['major', 'minor', 'revision'])

xapian_version = XAPIAN_VERSION(xapian.major_version(),
                                  xapian.minor_version(),
                                  xapian.revision())

default_parser_flags = (QueryParser.FLAG_PHRASE |
                        QueryParser.FLAG_BOOLEAN |
                        QueryParser.FLAG_LOVEHATE |
                        QueryParser.FLAG_SPELLING_CORRECTION |
                        QueryParser.FLAG_BOOLEAN_ANY_CASE |
                        QueryParser.FLAG_WILDCARD)


def _schema_name(schema):
    return "%s.%s" % (schema.__module__, schema.__name__)


def defaults(head, limit=0, mlimit=0, klimit=1.0, kmlimit=1.0):
    return (head, limit, mlimit, klimit, kmlimit)


def _lookup_schema(name):
    modname, expr = name.rsplit('.', 1)
    local_name = modname.split('.')[-1]
    mod = __import__(modname, {}, {}, local_name)
    return eval(expr, mod.__dict__)


def _prefix(name):
    return (u'X%s:' % name.upper()).encode('utf-8')


def to_term(value, prefix=None):
    value = value.lower()
    return _prefix(prefix) + value if prefix else value


class Record(object):
    """Nice attribute-accessable record for a search result."""

    def __init__(self, document, percent, rank, weight, query, db):
        self._xodb_document = document
        self._xodb_percent = percent
        self._xodb_rank = rank
        self._xodb_weight = weight
        self._xodb_query = query
        self._xodb_db = db

    @lazy_property
    def _xodb_schema(self):
        typ, data = loads(self._xodb_document.get_data())
        return _lookup_schema(typ).from_flat(data)

    def __getattr__(self, name):
        try:
            return self._xodb_schema[name].value
        except KeyError:
            raise AttributeError(name)

    def __repr__(self):
        return repr(self._xodb_schema)<commit_msg>Optimization to short circuit record attribut look up if the object is unloaded and the value is available as a value slot getter.<commit_after>import time
import string
import logging
from collections import namedtuple, OrderedDict
from contextlib import contextmanager

import xapian
import nilsimsa

from operator import itemgetter
from functools import partial
from json import loads

from xapian import Query, QueryParser, DocNotFoundError

from . import snowball
from .elements import Schema
from .exc import ValidationError, PrefixError
from .tools import LRUDict, lazy_property


RETRY_LIMIT = 5
RETRY_BACKOFF_FACTOR = 0.2

logger = logging.getLogger(__name__)

XAPIAN_VERSION = namedtuple('XAPIAN_VERSION', ['major', 'minor', 'revision'])

xapian_version = XAPIAN_VERSION(xapian.major_version(),
                                  xapian.minor_version(),
                                  xapian.revision())

default_parser_flags = (QueryParser.FLAG_PHRASE |
                        QueryParser.FLAG_BOOLEAN |
                        QueryParser.FLAG_LOVEHATE |
                        QueryParser.FLAG_SPELLING_CORRECTION |
                        QueryParser.FLAG_BOOLEAN_ANY_CASE |
                        QueryParser.FLAG_WILDCARD)


def _schema_name(schema):
    return "%s.%s" % (schema.__module__, schema.__name__)


def defaults(head, limit=0, mlimit=0, klimit=1.0, kmlimit=1.0):
    return (head, limit, mlimit, klimit, kmlimit)


def _lookup_schema(name):
    modname, expr = name.rsplit('.', 1)
    local_name = modname.split('.')[-1]
    mod = __import__(modname, {}, {}, local_name)
    return eval(expr, mod.__dict__)


def _prefix(name):
    return (u'X%s:' % name.upper()).encode('utf-8')


def to_term(value, prefix=None):
    value = value.lower()
    return _prefix(prefix) + value if prefix else value


class Record(object):
    """Nice attribute-accessable record for a search result."""

    def __init__(self, document, percent, rank, weight, query, db):
        self._xodb_document = document
        self._xodb_percent = percent
        self._xodb_rank = rank
        self._xodb_weight = weight
        self._xodb_query = query
        self._xodb_db = db
        self._loaded = False

    @lazy_property
    def _xodb_schema(self):
        typ, data = loads(self._xodb_document.get_data())
        self._loaded = True
        return _lookup_schema(typ).from_flat(data)

    def __getattr__(self, name):
        if not self._loaded:
            # short circuit expensive schema loading,
            # if value is available
            sort = self._xodb_db.value_sorts.get(name)
            # TODO: date, datetime
            if sort and sort in ('integer', 'string'):
                num = self._xodb_db.values[name]
                val = self._xodb_document.get_value(num)
                if sort == 'integer':
                    val = xapian.sortable_unserialise(val)
                return val
        try:
            return self._xodb_schema[name].value
        except KeyError:
            raise AttributeError(name)

    def __repr__(self):
        return repr(self._xodb_schema)
{'fraction_non_alphanumeric': 0.09081119885139986, 'fraction_numerical': 0.004307250538406317, 'mean_word_length': 2.671277997364954, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3345222', 'n_tokens_mistral': 1703, 'n_tokens_neox': 1622, 'n_words': 412}