<commit_before>r"""
Search objects (:mod: `qiita_db.search`)
====================================

..currentmodule:: qiita_db.search

This module provides functionality for searching studies and samples contained
in the qiita database. All language processing and querying of the database is
contained within each object.

Classes
-------

..autosummary::
    :toctree: generated/

    QiitaStudySearch

Examples
--------
Searches are done using boolean language, with AND, OR, and NOT supported,
as well as ordering through parenthesis. You can search over metadata using the
following operators::

>  <  =  <=  >=  includes

The operators act as they normally do, with includes used for substring
searches.

The object itself is used to search using the call method. In this
example, we will use the complex query::

(sample_type = ENVO:soil AND COMMON_NAME = "rhizosphere metagenome") AND
NOT Description_duplicate includes Burmese

>>> from qiita_db.search import QiitaStudySearch # doctest: +SKIP
>>> search = QiitaStudySearch() # doctest: +SKIP
>>> res, meta = search('(sample_type = ENVO:soil AND COMMON_NAME = '
...                    '"rhizosphere metagenome") AND NOT '
...                    'Description_duplicate includes Burmese',
...                    "<EMAIL>") # doctest: +SKIP
>>> print(res) # doctest: +SKIP
{1: ['SKM4.640180', 'SKB4.640189', 'SKB5.640181', 'SKB6.640176',
     'SKM5.640177', 'SKD4.640185', 'SKD6.640190', 'SKM6.640187',
     'SKD5.640186']}
>>> print(meta) # doctest: +SKIP
{"sample_type", "COMMON_NAME", "Description_duplicate"}

Note that the userid performing the search must also be passed, so the search
knows what studies are accessable.
"""

# -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------
from pyparsing import (alphas, nums, Word, dblQuotedString, oneOf, Optional,
                       opAssoc, CaselessLiteral, removeQuotes, Group,
                       operatorPrecedence, stringEnd)

from qiita_db.util import scrub_data, typecast_string, get_table_cols
from qiita_db.sql_connection import SQLConnectionHandler
from qiita_db.study import Study
from qiita_db.user import User

# column names from required_sample_info table
required_cols = {"study_id", "sample_id", "physical_location",
                 "has_physical_specimen", "has_extracted_data", "sample_type",
                 "collection_timestamp", "host_subject_id", "description"}
# required_cols = set(get_table_cols("required_sample_info"))


# classes to be constructed at parse time, from intermediate ParseResults
class UnaryOperation(object):
    def __init__(self, t):
        self.op, self.a = t[0]


class BinaryOperation(object):
    def __init__(self, t):
        self.op = t[0][1]
        self.operands = t[0][0::2]


class SearchAnd(BinaryOperation):
    def generate_sql(self):
        return "(%s)" % " AND ".join(oper.generate_sql()
                                     for oper in self.operands)

    def __repr__(self):
        return "AND:(%s)" % (",".join(str(oper) for oper in self.operands))


class SearchOr(BinaryOperation):
    def generate_sql(self):
        return "(%s)" % " OR ".join(oper.generate_sql()
                                    for oper in self.operands)

    def __repr__(self):
        return "OR:(%s)" % (",".join(str(oper) for oper in self.operands))


class SearchNot(UnaryOperation):
    def generate_sql(self):
        return "NOT %s" % self.a.generate_sql()

    def __repr__(self):
        return "NOT:(%s)" % str(self.a)


class SearchTerm(object):
    def __init__(self, tokens):
        self.term = tokens[0]
        # clean all the inputs
        for pos, term in enumerate(self.term):
            self.term[pos] = scrub_data(term)

    def generate_sql(self):
        # we can assume that the metadata is either in required_sample_info
        # or the study-specific table
        if self.term[0] in required_cols:
            self.term[0] = "r.%s" % self.term[0].lower()
        else:
            self.term[0] = "s.%s" % self.term[0].lower()

        if self.term[1] == "includes":
            # substring search, so create proper query for it
            return "%s LIKE '%%%s%%'" % (self.term[0], self.term[2])
        else:
            # standard query so just return it, adding quotes if string
            if isinstance(typecast_string(self.term[2]), str):
                self.term[2] = ''.join(("'", self.term[2], "'"))
            return ' '.join(self.term)

    def __repr__(self):
        if self.term[1] == "includes":
            return "%s LIKE '%%%s%%')" % (self.term[0], self.term[2])
        else:
            return ' '.join(self.term)


class QiitaStudySearch(object):
    """QiitaStudySearch object to parse and run searches on studies."""
    def __call__(self, searchstr, user):
        """Runs a Study query and returns matching studies and samples

        Parameters
        ----------
        searchstr : str
            Search string to use
        user : str
            User making the search. Needed for permissions checks.

        Returns
        -------
        dict
            Found samples in format {study_id: [samp_id1, samp_id2,...]}
        set
            metadata column names searched for

        Notes
        -----
        Metadata column names and string searches are case-sensitive
        """
        study_sql, sample_sql, meta_headers = \
            self._parse_study_search_string(searchstr)
        conn_handler = SQLConnectionHandler()
        # get all studies containing the metadata headers requested
        study_ids = {x[0] for x in conn_handler.execute_fetchall(study_sql)}
        # strip to only studies user has access to
        userobj = User(user)
        study_ids = study_ids.intersection(Study.get_public() +
                                           userobj.private_studies +
                                           userobj.shared_studies)
        results = {}
        # run search on each study to get out the matching samples
        for sid in study_ids:

            results[sid] = conn_handler.execute_fetchall(
                sample_sql.format(sid))
        return results, meta_headers

    def _parse_study_search_string(self, searchstr):
        """parses string into SQL query for study search

        Parameters
        ----------<commit_msg>update docstrings for new change<commit_after>r"""
Search objects (:mod: `qiita_db.search`)
====================================

..currentmodule:: qiita_db.search

This module provides functionality for searching studies and samples contained
in the qiita database. All language processing and querying of the database is
contained within each object.

Classes
-------

..autosummary::
    :toctree: generated/

    QiitaStudySearch

Examples
--------
Searches are done using boolean language, with AND, OR, and NOT supported,
as well as ordering through parenthesis. You can search over metadata using the
following operators::

>  <  =  <=  >=  includes

The operators act as they normally do, with includes used for substring
searches.

The object itself is used to search using the call method. In this
example, we will use the complex query::

(sample_type = ENVO:soil AND COMMON_NAME = "rhizosphere metagenome") AND
NOT Description_duplicate includes Burmese

>>> from qiita_db.search import QiitaStudySearch # doctest: +SKIP
>>> search = QiitaStudySearch() # doctest: +SKIP
>>> res, meta = search('(sample_type = ENVO:soil AND COMMON_NAME = '
...                    '"rhizosphere metagenome") AND NOT '
...                    'Description_duplicate includes Burmese',
...                    "<EMAIL>") # doctest: +SKIP
>>> print(res) # doctest: +SKIP
{1: [['SKM4.640180', 'rhizosphere metagenome', 'Bucu Rhizo', 'ENVO:soil'],
     ['SKM5.640177', 'rhizosphere metagenome', 'Bucu Rhizo', 'ENVO:soil'],
     ['SKD4.640185', 'rhizosphere metagenome', 'Diesel Rhizo', 'ENVO:soil'],
     ['SKD6.640190', 'rhizosphere metagenome', 'Diesel Rhizo', 'ENVO:soil'],
     ['SKM6.640187', 'rhizosphere metagenome', 'Bucu Rhizo', 'ENVO:soil'],
     ['SKD5.640186', 'rhizosphere metagenome', 'Diesel Rhizo', 'ENVO:soil']]}
>>> print(meta) # doctest: +SKIP
["COMMON_NAME", "Description_duplicate", "sample_type"]

Note that the userid performing the search must also be passed, so the search
knows what studies are accessable.
"""

# -----------------------------------------------------------------------------
# Copyright (c) 2014--, The Qiita Development Team.
#
# Distributed under the terms of the BSD 3-clause License.
#
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------
from pyparsing import (alphas, nums, Word, dblQuotedString, oneOf, Optional,
                       opAssoc, CaselessLiteral, removeQuotes, Group,
                       operatorPrecedence, stringEnd)

from qiita_db.util import scrub_data, typecast_string, get_table_cols
from qiita_db.sql_connection import SQLConnectionHandler
from qiita_db.study import Study
from qiita_db.user import User

# column names from required_sample_info table
required_cols = {"study_id", "sample_id", "physical_location",
                 "has_physical_specimen", "has_extracted_data", "sample_type",
                 "collection_timestamp", "host_subject_id", "description"}
# required_cols = set(get_table_cols("required_sample_info"))


# classes to be constructed at parse time, from intermediate ParseResults
class UnaryOperation(object):
    def __init__(self, t):
        self.op, self.a = t[0]


class BinaryOperation(object):
    def __init__(self, t):
        self.op = t[0][1]
        self.operands = t[0][0::2]


class SearchAnd(BinaryOperation):
    def generate_sql(self):
        return "(%s)" % " AND ".join(oper.generate_sql()
                                     for oper in self.operands)

    def __repr__(self):
        return "AND:(%s)" % (",".join(str(oper) for oper in self.operands))


class SearchOr(BinaryOperation):
    def generate_sql(self):
        return "(%s)" % " OR ".join(oper.generate_sql()
                                    for oper in self.operands)

    def __repr__(self):
        return "OR:(%s)" % (",".join(str(oper) for oper in self.operands))


class SearchNot(UnaryOperation):
    def generate_sql(self):
        return "NOT %s" % self.a.generate_sql()

    def __repr__(self):
        return "NOT:(%s)" % str(self.a)


class SearchTerm(object):
    def __init__(self, tokens):
        self.term = tokens[0]
        # clean all the inputs
        for pos, term in enumerate(self.term):
            self.term[pos] = scrub_data(term)

    def generate_sql(self):
        # we can assume that the metadata is either in required_sample_info
        # or the study-specific table
        if self.term[0] in required_cols:
            self.term[0] = "r.%s" % self.term[0].lower()
        else:
            self.term[0] = "s.%s" % self.term[0].lower()

        if self.term[1] == "includes":
            # substring search, so create proper query for it
            return "%s LIKE '%%%s%%'" % (self.term[0], self.term[2])
        else:
            # standard query so just return it, adding quotes if string
            if isinstance(typecast_string(self.term[2]), str):
                self.term[2] = ''.join(("'", self.term[2], "'"))
            return ' '.join(self.term)

    def __repr__(self):
        if self.term[1] == "includes":
            return "%s LIKE '%%%s%%')" % (self.term[0], self.term[2])
        else:
            return ' '.join(self.term)


class QiitaStudySearch(object):
    """QiitaStudySearch object to parse and run searches on studies."""
    def __call__(self, searchstr, user):
        """Runs a Study query and returns matching studies and samples

        Parameters
        ----------
        searchstr : str
            Search string to use
        user : str
            User making the search. Needed for permissions checks.

        Returns
        -------
        dict
            Found samples in format 
            {study_id: [[samp_id1, meta1, meta2, ...],
                        [samp_id2, meta1, meta2, ...], ...}
        list
            metadata column names searched for

        Notes
        -----
        Metadata information for each sample is in the same order as the
        metadata columns list returned
        
        Metadata column names and string searches are case-sensitive
        """
        study_sql, sample_sql, meta_headers = \
            self._parse_study_search_string(searchstr)
        conn_handler = SQLConnectionHandler()
        # get all studies containing the metadata headers requested
        study_ids = {x[0] for x in conn_handler.execute_fetchall(study_sql)}
        # strip to only studies user has access to
        userobj = User(user)
        study_ids = study_ids.intersection(Study.get_public() +
                                           userobj.private_studies +
                                           userobj.shared_studies)
        results = {}
        # run search on each study to get out the matching samples
        for sid in study_ids:

            results[sid] = conn_handler.execute_fetchall(
                sample_sql.format(sid))
        return results, meta_headers

    def _parse_study_search_string(self, searchstr):
        """parses string into SQL query for study search

        Parameters
        ----------
{'fraction_non_alphanumeric': 0.1367278554778555, 'fraction_numerical': 0.01231060606060606, 'mean_word_length': 2.3559032021510635, 'pattern_counts': {'":': 4, '<': 9, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3903883', 'n_tokens_mistral': 4039, 'n_tokens_neox': 3833, 'n_words': 1181}