<commit_before>"""
A subclass of tiddlywebplugins.sqlalchemy with mysql specific
tune ups.
"""
from __future__ import absolute_import

import logging

from sqlalchemy.engine import create_engine
from sqlalchemy.exc import ArgumentError, NoSuchTableError
from sqlalchemy.sql.expression import and_, or_, text as text_, alias
from sqlalchemy.sql import func
from sqlalchemy.schema import Table, PrimaryKeyConstraint
from sqlalchemy.orm import mapper

from tiddlyweb.model.tiddler import Tiddler
from tiddlyweb.serializer import Serializer
from tiddlyweb.store import StoreError

from tiddlywebplugins.sqlalchemy import (Store as SQLStore,
        sRevision, metadata, Session,
        field_table, revision_table, bag_table, policy_table,
        recipe_table, principal_table, role_table, user_table)

from tiddlyweb.filters import FilterIndexRefused

from pyparsing import (printables, alphanums, OneOrMore, Group,
        Combine, Suppress, Optional, FollowedBy, Literal, CharsNotIn,
        Word, Keyword, Empty, White, Forward, QuotedString, StringEnd,
        ParseException)

#import logging
#logging.basicConfig()
#logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
# logging.getLogger('sqlalchemy.orm.unitofwork').setLevel(logging.DEBUG)


ENGINE = None
MAPPED = False
TABLES = [field_table, revision_table, bag_table, policy_table, recipe_table,
        principal_table, role_table, user_table]

class Store(SQLStore):

    def _init_store(self):
        """
        Establish the database engine and session,
        creating tables if needed.
        """
        global ENGINE, MAPPED
        if not ENGINE:
            ENGINE = create_engine(self._db_config(), pool_recycle=3600)
        metadata.bind = ENGINE
        Session.configure(bind=ENGINE)
        self.session = Session()
        self.serializer = Serializer('text')
        self.parser = DEFAULT_PARSER
        self.producer = Producer()

        if not MAPPED:
            for table in TABLES:
                table.kwargs['mysql_charset'] = 'utf8'
            metadata.create_all(ENGINE)
            MAPPED = True

    def search(self, search_query=''):
        try:
            subquery = (self.session.query(func.max(sRevision.number))
                    .group_by(sRevision.bag_name, sRevision.tiddler_title)
                    .subquery())
            
            query = (self.session.query(sRevision.bag_name,
                    sRevision.tiddler_title, sRevision.number)
                    .group_by(sRevision.bag_name, sRevision.tiddler_title))

            try:
                ast = self.parser(search_query)[0]
                query = self.producer.produce(ast, query)
            except ParseException, exc:
                raise StoreError('failed to parse search query: %s' % exc)

            query = query.filter(sRevision.number.in_(subquery))

            return (Tiddler(unicode(stiddler.tiddler_title),
                unicode(stiddler.bag_name)) for stiddler in query.all())
        except:
            self.session.rollback()
            raise


def index_query(environ, **kwargs):<commit_msg>Add the head view back in as it is the most efficient way to get the current rev of any tiddler.<commit_after>"""
A subclass of tiddlywebplugins.sqlalchemy with mysql specific
tune ups.
"""
from __future__ import absolute_import

import logging

from sqlalchemy.engine import create_engine
from sqlalchemy.exc import ArgumentError, NoSuchTableError
from sqlalchemy.sql.expression import and_, or_, text as text_, alias
from sqlalchemy.sql import func
from sqlalchemy.schema import Table, PrimaryKeyConstraint
from sqlalchemy.orm import mapper

from tiddlyweb.model.tiddler import Tiddler
from tiddlyweb.serializer import Serializer
from tiddlyweb.store import StoreError

from tiddlywebplugins.sqlalchemy import (Store as SQLStore,
        sRevision, metadata, Session,
        field_table, revision_table, bag_table, policy_table,
        recipe_table, principal_table, role_table, user_table)

from tiddlyweb.filters import FilterIndexRefused

from pyparsing import (printables, alphanums, OneOrMore, Group,
        Combine, Suppress, Optional, FollowedBy, Literal, CharsNotIn,
        Word, Keyword, Empty, White, Forward, QuotedString, StringEnd,
        ParseException)

#import logging
#logging.basicConfig()
#logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO)
# logging.getLogger('sqlalchemy.orm.unitofwork').setLevel(logging.DEBUG)


ENGINE = None
MAPPED = False
TABLES = [field_table, revision_table, bag_table, policy_table, recipe_table,
        principal_table, role_table, user_table]

class sHead(object):
    def __repr__(self):
        return '<sHead(%s:%s:%s)>' % (self.revision_bag_name,
                self.revision_tiddler_title, self.head_rev)


class Store(SQLStore):

    def _init_store(self):
        """
        Establish the database engine and session,
        creating tables if needed.
        """
        global ENGINE, MAPPED
        if not ENGINE:
            ENGINE = create_engine(self._db_config(), pool_recycle=3600)
        metadata.bind = ENGINE
        Session.configure(bind=ENGINE)
        self.session = Session()
        self.serializer = Serializer('text')
        self.parser = DEFAULT_PARSER
        self.producer = Producer()

        if not MAPPED:
            for table in TABLES:
                table.kwargs['mysql_charset'] = 'utf8'
            metadata.create_all(ENGINE)
            MAPPED = True


        try:
            self._load_head_table()
        except NoSuchTableError:
            print 'creating view HEAD'
            self.session.execute("""
CREATE VIEW head
  AS SELECT revision.bag_name AS revision_bag_name,
    revision.tiddler_title AS revision_tiddler_title,
    max(revision.number) AS head_rev
  FROM revision GROUP BY revision.bag_name, revision.tiddler_title;
""")
            self._load_head_table()

    def _load_head_table(self):
        head_table = Table('head', metadata,
                PrimaryKeyConstraint('head_rev'),
                autoload_with=ENGINE,
                useexisting=True,
                autoload=True)
        try:
            mapper(sHead, head_table)
        except ArgumentError:
            print 'remap HEAD'


    def search(self, search_query=''):
        try:
            
            query = (self.session.query(sRevision.bag_name,
                    sRevision.tiddler_title))
            query = query.filter(sHead.head_rev==sRevision.number)

            try:
                ast = self.parser(search_query)[0]
                query = self.producer.produce(ast, query)
            except ParseException, exc:
                raise StoreError('failed to parse search query: %s' % exc)

            return (Tiddler(unicode(stiddler.tiddler_title),
                unicode(stiddler.bag_name)) for stiddler in query.all())
        except:
            self.session.rollback()
            raise


def index_query(environ, **kwargs):
{'fraction_non_alphanumeric': 0.08326155612977318, 'fraction_numerical': 0.0017226528854435831, 'mean_word_length': 2.763911399243652, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3069822', 'n_tokens_mistral': 2095, 'n_tokens_neox': 1981, 'n_words': 491}