<commit_before># -*- coding: utf-8 -*-
"""
    Configuration

    A sphinx config generator for Tryton

    :copyright: (c) 2011 by <NAME>
    :license: BSD, see LICENSE for more details.
"""
from trytond.model import ModelSQL
from trytond.config import CONFIG
from jinja2 import Environment

from utils import guess_type


class BaseSource(object):
    """Sphinx allows inheritance in data sources. having the SQL settings for 
    each model object would be too redundant since the only changing attributes
    are the SQL query itself and the various attributes.
    """

    #: Name of a data source needs to be unique and the best way seems to be to
    #: resuse the _table attribute of tryton which is guaranteed to exists and
    #: be unique for every model that has a database representation (in other
    #: words every object other than wizards and reports).
    #:
    #: For the Base Source this defaults to `base_source` and any 
    #: :class:`DataSource` which inherits from :class:`BaseSource` will use the
    #: settings from this instance
    name = 'base_source'

    #: data source type. mandatory, no default value. The possible types are
    #: mysql, pgsql and this can be inferred from the config
    type = None

    #: All the following database settings (except :attr:`db`) can be obtained
    #: from `trytond.config`.
    sql_host = None
    sql_user = None
    sql_pass = None
    sql_db = None
    sql_port = None


    @classmethod
    def from_tryton_config(cls, database_name):
        """Creates a :class:`BaseSource` from the current tryton configuration
        in the environment.

        :param database_name: The database name to use in the database source.
                              Equivalent to setting :attr:`sql_db`
        """
        self = cls()

        self.type = {
            'postgresql': 'pgsql',
            'mysql': 'mysql',
            }.get(CONFIG.options['db_type'])
        self.sql_host = CONFIG.options['db_host']
        self.sql_port = CONFIG.options['db_port']

        self.sql_user = CONFIG.options['db_user']
        self.sql_pass = CONFIG.options['db_password']
        
        self.sql_db = database_name

        return self

    def as_string(self):
        """Returns the string representation of the config as it has to appear
        int he sphinx.conf
        """
        template = Environment().from_string("""
source {{name}}
{
    # data source type. mandatory, no default value
    # known types are mysql, pgsql, mssql, xmlpipe, xmlpipe2, odbc
    type        = {{type}}

    sql_host    = {{sql_host}}
    sql_user    = {{sql_user}}
    sql_pass    = {{sql_pass}}
    sql_db      = {{sql_db}}
    sql_port    = {{sql_port}}

    }
        """)
        return template.render(
            name=self.name,
            type=self.type,
            sql_host=self.sql_host,
            sql_user=self.sql_user,
            sql_pass=self.sql_pass,
            sql_db=self.sql_db,
            sql_port=self.sql_port
            )


class DataSource(object):
    """This class represents a `sphinx data source 
    <http://sphinxsearch.com/docs/1.10/sources.html>`_. 

    The data to be indexed can generally come from very different sources: 
    SQL databases, plain text files, HTML files, mailboxes, and so on. From 
    Sphinx point of view, the data it indexes is a set of structured documents,
    each of which has the same set of fields. This is biased towards SQL, where
    each row correspond to a document, and each column to a field.

    Int he case of tryton the data source basically represents a combination of
    the database backend (name and credentials), a `ModelSQL <>`_ object and
    its `_table` representation.


    The possible arguments are:

    `name`: Name of a data source. (needs to be unique)
            .. tip:: 
                the best way seems to be to resuse the _table attribute of 
                tryton which is guaranteed to exists and be unique for every 
                model that has a database representation (in other words every 
                object other than wizards and reports)

    `sql_query`: The main document fetch query. 
    `attributes`: A :class:`dict` of field names as keys and type as values
                  eg {'name': 'sql_field_string'}
    `sql_query_range` : Range query setup, query that must return min and max 
                        ID values optional, default is empty
    `sql_range_step`: Range query step (Default: 1024)
    """


    def __init__(self, name, sql_query, attributes, 
            sql_query_range=None, sql_range_step=1024):
        self.name = name
        self.sql_query = sql_query
        self.attributes = attributes
        self.sql_query_range = sql_query_range
        self.sql_range_step = sql_range_step

    @classmethod
    def from_model(cls, model_object, base_source = None):
        """Creates and returns a new data source from a given model

        :param model_object: The instance of a model as obtained from the 
                             `trytond.pool.Pool`
        :param base_source: The base_source to inherit from. Must be an 
                            instance of :class:`BaseSource`
        """
        assert isinstance(model_object, ModelSQL), \
            "model_object must be an instance of ModelSQL"
        if base_source:
            assert isinstance(base_source, BaseSource), \
                "base_source must be an instance of BaseSource (got %s)" % type(
                    base_source)

        attributes = {}
        for name, field in model_object._columns.iteritems():
            if field.select != 1:
                continue
            try:
                attributes[name] = guess_type(field)
            except ValueError:
                # Ignore if a value error is raised because its a field which
                # sphinx doesnt know the proper type for conversion
                continue

        sql_query = ""
        sql_query_range = 'SELECT MIN(id),MAX(id) FROM "%s"' \
            % model_object._table
 
        if attributes:
            # Construct the SQL query if attributes do exist
            sql_query = ['"SELECT "%s"."id" AS "id",' % model_object._table]<commit_msg>Remove the unnecessary "before SELECT<commit_after># -*- coding: utf-8 -*-
"""
    Configuration

    A sphinx config generator for Tryton

    :copyright: (c) 2011 by <NAME>
    :license: BSD, see LICENSE for more details.
"""
from trytond.model import ModelSQL
from trytond.config import CONFIG
from jinja2 import Environment

from utils import guess_type


class BaseSource(object):
    """Sphinx allows inheritance in data sources. having the SQL settings for 
    each model object would be too redundant since the only changing attributes
    are the SQL query itself and the various attributes.
    """

    #: Name of a data source needs to be unique and the best way seems to be to
    #: resuse the _table attribute of tryton which is guaranteed to exists and
    #: be unique for every model that has a database representation (in other
    #: words every object other than wizards and reports).
    #:
    #: For the Base Source this defaults to `base_source` and any 
    #: :class:`DataSource` which inherits from :class:`BaseSource` will use the
    #: settings from this instance
    name = 'base_source'

    #: data source type. mandatory, no default value. The possible types are
    #: mysql, pgsql and this can be inferred from the config
    type = None

    #: All the following database settings (except :attr:`db`) can be obtained
    #: from `trytond.config`.
    sql_host = None
    sql_user = None
    sql_pass = None
    sql_db = None
    sql_port = None


    @classmethod
    def from_tryton_config(cls, database_name):
        """Creates a :class:`BaseSource` from the current tryton configuration
        in the environment.

        :param database_name: The database name to use in the database source.
                              Equivalent to setting :attr:`sql_db`
        """
        self = cls()

        self.type = {
            'postgresql': 'pgsql',
            'mysql': 'mysql',
            }.get(CONFIG.options['db_type'])
        self.sql_host = CONFIG.options['db_host']
        self.sql_port = CONFIG.options['db_port']

        self.sql_user = CONFIG.options['db_user']
        self.sql_pass = CONFIG.options['db_password']
        
        self.sql_db = database_name

        return self

    def as_string(self):
        """Returns the string representation of the config as it has to appear
        int he sphinx.conf
        """
        template = Environment().from_string("""
source {{name}}
{
    # data source type. mandatory, no default value
    # known types are mysql, pgsql, mssql, xmlpipe, xmlpipe2, odbc
    type        = {{type}}

    sql_host    = {{sql_host}}
    sql_user    = {{sql_user}}
    sql_pass    = {{sql_pass}}
    sql_db      = {{sql_db}}
    sql_port    = {{sql_port}}

    }
        """)
        return template.render(
            name=self.name,
            type=self.type,
            sql_host=self.sql_host,
            sql_user=self.sql_user,
            sql_pass=self.sql_pass,
            sql_db=self.sql_db,
            sql_port=self.sql_port
            )


class DataSource(object):
    """This class represents a `sphinx data source 
    <http://sphinxsearch.com/docs/1.10/sources.html>`_. 

    The data to be indexed can generally come from very different sources: 
    SQL databases, plain text files, HTML files, mailboxes, and so on. From 
    Sphinx point of view, the data it indexes is a set of structured documents,
    each of which has the same set of fields. This is biased towards SQL, where
    each row correspond to a document, and each column to a field.

    Int he case of tryton the data source basically represents a combination of
    the database backend (name and credentials), a `ModelSQL <>`_ object and
    its `_table` representation.


    The possible arguments are:

    `name`: Name of a data source. (needs to be unique)
            .. tip:: 
                the best way seems to be to resuse the _table attribute of 
                tryton which is guaranteed to exists and be unique for every 
                model that has a database representation (in other words every 
                object other than wizards and reports)

    `sql_query`: The main document fetch query. 
    `attributes`: A :class:`dict` of field names as keys and type as values
                  eg {'name': 'sql_field_string'}
    `sql_query_range` : Range query setup, query that must return min and max 
                        ID values optional, default is empty
    `sql_range_step`: Range query step (Default: 1024)
    """


    def __init__(self, name, sql_query, attributes, 
            sql_query_range=None, sql_range_step=1024):
        self.name = name
        self.sql_query = sql_query
        self.attributes = attributes
        self.sql_query_range = sql_query_range
        self.sql_range_step = sql_range_step

    @classmethod
    def from_model(cls, model_object, base_source = None):
        """Creates and returns a new data source from a given model

        :param model_object: The instance of a model as obtained from the 
                             `trytond.pool.Pool`
        :param base_source: The base_source to inherit from. Must be an 
                            instance of :class:`BaseSource`
        """
        assert isinstance(model_object, ModelSQL), \
            "model_object must be an instance of ModelSQL"
        if base_source:
            assert isinstance(base_source, BaseSource), \
                "base_source must be an instance of BaseSource (got %s)" % type(
                    base_source)

        attributes = {}
        for name, field in model_object._columns.iteritems():
            if field.select != 1:
                continue
            try:
                attributes[name] = guess_type(field)
            except ValueError:
                # Ignore if a value error is raised because its a field which
                # sphinx doesnt know the proper type for conversion
                continue

        sql_query = ""
        sql_query_range = 'SELECT MIN(id),MAX(id) FROM "%s"' \
            % model_object._table
 
        if attributes:
            # Construct the SQL query if attributes do exist
            sql_query = ['SELECT "%s"."id" AS "id",' % model_object._table]
{'fraction_non_alphanumeric': 0.07726145725025994, 'fraction_numerical': 0.003039270575061985, 'mean_word_length': 2.220190574298223, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5069532', 'n_tokens_mistral': 3453, 'n_tokens_neox': 3243, 'n_words': 1285}