<commit_before>from datetime import datetime
import pyes

conn = None
index = None

TYPE = 'annotation'
MAPPING = {
    'annotator_schema_version': {'type': 'string', 'null_value': 'v1.0'},
    'created': {'type': 'date'},
    'updated': {'type': 'date'},
    'quote': {'type': 'string'},
    'tags': {'type': 'string', 'index_name': 'tag'},
    'text': {'type': 'string'},
    'uri': {'type': 'string', 'index': 'not_analyzed'},
    'user' : {'type': 'string', 'index' : 'not_analyzed'},
    'ranges': {
        'index_name': 'range',
        'properties': {
            'start': {'type': 'string', 'index': 'not_analyzed'},
            'end':   {'type': 'string', 'index': 'not_analyzed'},
            'startOffset': {'type': 'integer'},
            'endOffset':   {'type': 'integer'},
        }
    }
}

def configure(c, config):
    global conn, index
    conn = c
    index = config['ELASTICSEARCH_INDEX']

class ValidationError(Exception):
    pass

class Annotation(dict):

    # It would be lovely if this were called 'get', but the dict semantics
    # already define that method name.
    @classmethod
    def fetch(cls, id):
        try:
            doc = conn.get(index, TYPE, id)
        # We should be more specific than this, but pyes doesn't raise the
        # correct exception for missing documents at the moment
        except pyes.exceptions.ElasticSearchException:
            return None
        return Annotation(doc['_source'], id=id)

    @classmethod
    def _build_query(cls, offset=0, limit=20, **kwargs):
        f = {'and': []}
        q = {'filtered': {'query': {'match_all': {}}, 'filter': f}}

        uid = kwargs.pop('_user_id', None)

        for k, v in kwargs.iteritems():
            q['filtered']['filter']['and'].append({'term': {k: v}})

        # Append permissions filter
        uidq = {'missing': {'field': 'permissions.read'}}

        if uid:
            user_readable = {'term': {'permissions.read': uid}}
            uidq = {'or': [uidq, user_readable]}

        q['filtered']['filter']['and'].append(uidq)

        return {
            # Sort most recent first
            'sort': [{'updated': {'order': 'desc'}}],
            'from': offset,
            'size': limit,
            'query': q
        }

    @classmethod
    def search(cls, **kwargs):
        q = cls._build_query(**kwargs)
        res = conn.search(q, index, TYPE)<commit_msg>Clean up last commit and add some explanatory comments<commit_after>from datetime import datetime
import pyes

conn = None
index = None

TYPE = 'annotation'
MAPPING = {
    'annotator_schema_version': {'type': 'string', 'null_value': 'v1.0'},
    'created': {'type': 'date'},
    'updated': {'type': 'date'},
    'quote': {'type': 'string'},
    'tags': {'type': 'string', 'index_name': 'tag'},
    'text': {'type': 'string'},
    'uri': {'type': 'string', 'index': 'not_analyzed'},
    'user' : {'type': 'string', 'index' : 'not_analyzed'},
    'ranges': {
        'index_name': 'range',
        'properties': {
            'start': {'type': 'string', 'index': 'not_analyzed'},
            'end':   {'type': 'string', 'index': 'not_analyzed'},
            'startOffset': {'type': 'integer'},
            'endOffset':   {'type': 'integer'},
        }
    }
}

def configure(c, config):
    global conn, index
    conn = c
    index = config['ELASTICSEARCH_INDEX']

class ValidationError(Exception):
    pass

class Annotation(dict):

    # It would be lovely if this were called 'get', but the dict semantics
    # already define that method name.
    @classmethod
    def fetch(cls, id):
        try:
            doc = conn.get(index, TYPE, id)
        # We should be more specific than this, but pyes doesn't raise the
        # correct exception for missing documents at the moment
        except pyes.exceptions.ElasticSearchException:
            return None
        return Annotation(doc['_source'], id=id)

    @classmethod
    def _build_query(cls, offset=0, limit=20, _user_id=None, **kwargs):
        # Base query is a filtered match_all
        f = {'and': []}
        q = {'filtered': {'query': {'match_all': {}}, 'filter': f}}

        # Add a term query for each kwarg that isn't otherwise accounted for
        for k, v in kwargs.iteritems():
            q['filtered']['filter']['and'].append({'term': {k: v}})

        # Append permissions filter
        user_q = {'missing': {'field': 'permissions.read'}}

        # We match annotations which *either* lack the 'permissions.read'
        # field, *or* which contain the current user in said field.
        if _user_id:
            user_readable = {'term': {'permissions.read': _user_id}}
            user_q = {'or': [user_q, user_readable]}

        q['filtered']['filter']['and'].append(user_q)

        return {
            'sort': [{'updated': {'order': 'desc'}}], # Sort most recent first
            'from': offset,
            'size': limit,
            'query': q
        }

    @classmethod
    def search(cls, **kwargs):
        q = cls._build_query(**kwargs)
        res = conn.search(q, index, TYPE)
{'fraction_non_alphanumeric': 0.16929444225463144, 'fraction_numerical': 0.001970831690973591, 'mean_word_length': 2.573943661971831, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7123162', 'n_tokens_mistral': 1544, 'n_tokens_neox': 1507, 'n_words': 445}