<commit_before>from itertools import count
import logging
import inspect
import copy
import re
from .settings import config
from .errors import DataJointError
from .fetch import Fetch, Fetch1
from .preview import preview, repr_html
from .condition import AndList, Not, \
    make_condition, assert_join_compatibility, extract_column_names, PromiscuousOperand

logger = logging.getLogger(__name__)


class QueryExpression:
    """
    QueryExpression implements query operators to derive new entity set from its input.
    A QueryExpression object generates a SELECT statement in SQL.
    QueryExpression operators are restrict, join, proj, aggr, and union.

    A QueryExpression object has a support, a restriction (an AndList), and heading.
    Property `heading` (type dj.Heading) contains information about the attributes.
    It is loaded from the database and updated by proj.

    Property `support` is the list of table names or other QueryExpressions to be joined.

    The restriction is applied first without having access to the attributes generated by the projection.
    Then projection is applied by selecting modifying the heading attribute.

    Application of operators does not always lead to the creation of a subquery.
    A subquery is generated when:
        1. A restriction is applied on any computed or renamed attributes
        2. A projection is applied remapping remapped attributes
        3. Subclasses: Join, Aggregation, and Union have additional specific rules.
    """

    _restriction = None
    _restriction_attributes = None

    # subclasses or instantiators must provide values
    _connection = None
    _heading = None
    _support = None
    _join_attributes = []

    @property
    def connection(self):
        """ a dj.Connection object """
        assert self._connection is not None
        return self._connection

    @property
    def support(self):
        """ A list of table names or subqueries to from the FROM clause """
        assert self._support is not None
        return self._support

    @property
    def heading(self):
        """ a dj.Heading object, reflects the effects of the projection operator .proj """
        return self._heading

    @property
    def restriction(self):
        """ a AndList object of restrictions applied to input to produce the result """
        if self._restriction is None:
            self._restriction = AndList()
        return self._restriction
    
    @property
    def restriction_attributes(self):
        """ the set of names invoked in the WHERE clause """
        if self._restriction_attributes is None:
            self._restriction_attributes = set()
        return self._restriction_attributes

    @property
    def primary_key(self):
        return self.heading.primary_key

    __subquery_alias_count = count()    # count for alias names used in from_clause

    def from_clause(self, left=False):
        support = ('(' + src.make_sql() + ') as `_s%x`' % next(
            self.__subquery_alias_count) if isinstance(src, QueryExpression) else src for src in self.support)
        clause = next(support)
        for s, a in zip(support, self._join_attributes):
            clause += '{left} JOIN {clause}{using}'.format(
                left=" LEFT" if left else "",
                clause=s,
                using="" if not a else " USING (%s)" % ",".join('`%s`' % _ for _ in a))
        return clause

    @property
    def where_clause(self):
        return '' if not self.restriction else ' WHERE(%s)' % ')AND('.join(str(s) for s in self.restriction)

    def make_sql(self, fields=None, left=False):
        """
        Make the SQL SELECT statement.
        :param fields: used to explicitly set the select attributes
        """
        distinct = self.heading.names == self.primary_key
        return 'SELECT {distinct}{fields} FROM {from_}{where}'.format(
            distinct="DISTINCT " if distinct else "",
            fields=self.heading.as_sql(fields or self.heading.names),
            from_=self.from_clause(), where=self.where_clause)

    # --------- query operators -----------
    def make_subquery(self):
        """ create a new SELECT statement where self is the FROM clause """<commit_msg>bugfix in outer union<commit_after>from itertools import count
import logging
import inspect
import copy
import re
from .settings import config
from .errors import DataJointError
from .fetch import Fetch, Fetch1
from .preview import preview, repr_html
from .condition import AndList, Not, \
    make_condition, assert_join_compatibility, extract_column_names, PromiscuousOperand

logger = logging.getLogger(__name__)


class QueryExpression:
    """
    QueryExpression implements query operators to derive new entity set from its input.
    A QueryExpression object generates a SELECT statement in SQL.
    QueryExpression operators are restrict, join, proj, aggr, and union.

    A QueryExpression object has a support, a restriction (an AndList), and heading.
    Property `heading` (type dj.Heading) contains information about the attributes.
    It is loaded from the database and updated by proj.

    Property `support` is the list of table names or other QueryExpressions to be joined.

    The restriction is applied first without having access to the attributes generated by the projection.
    Then projection is applied by selecting modifying the heading attribute.

    Application of operators does not always lead to the creation of a subquery.
    A subquery is generated when:
        1. A restriction is applied on any computed or renamed attributes
        2. A projection is applied remapping remapped attributes
        3. Subclasses: Join, Aggregation, and Union have additional specific rules.
    """

    _restriction = None
    _restriction_attributes = None

    # subclasses or instantiators must provide values
    _connection = None
    _heading = None
    _support = None
    _join_attributes = []

    @property
    def connection(self):
        """ a dj.Connection object """
        assert self._connection is not None
        return self._connection

    @property
    def support(self):
        """ A list of table names or subqueries to from the FROM clause """
        assert self._support is not None
        return self._support

    @property
    def heading(self):
        """ a dj.Heading object, reflects the effects of the projection operator .proj """
        return self._heading

    @property
    def restriction(self):
        """ a AndList object of restrictions applied to input to produce the result """
        if self._restriction is None:
            self._restriction = AndList()
        return self._restriction
    
    @property
    def restriction_attributes(self):
        """ the set of names invoked in the WHERE clause """
        if self._restriction_attributes is None:
            self._restriction_attributes = set()
        return self._restriction_attributes

    @property
    def primary_key(self):
        return self.heading.primary_key

    __subquery_alias_count = count()    # count for alias names used in from_clause

    def from_clause(self, left=False):
        support = ('(' + src.make_sql() + ') as `_s%x`' % next(
            self.__subquery_alias_count) if isinstance(src, QueryExpression) else src for src in self.support)
        clause = next(support)
        for s, a in zip(support, self._join_attributes):
            clause += '{left} JOIN {clause}{using}'.format(
                left=" LEFT" if left else "",
                clause=s,
                using="" if not a else " USING (%s)" % ",".join('`%s`' % _ for _ in a))
        return clause

    @property
    def where_clause(self):
        return '' if not self.restriction else ' WHERE(%s)' % ')AND('.join(str(s) for s in self.restriction)

    def make_sql(self, fields=None, left=False):
        """
        Make the SQL SELECT statement.
        :param fields: used to explicitly set the select attributes
        """
        distinct = self.heading.names == self.primary_key
        return 'SELECT {distinct}{fields} FROM {from_}{where}'.format(
            distinct="DISTINCT " if distinct else "",
            fields=self.heading.as_sql(fields or self.heading.names),
            from_=self.from_clause(left=left), where=self.where_clause)

    # --------- query operators -----------
    def make_subquery(self):
        """ create a new SELECT statement where self is the FROM clause """
{'fraction_non_alphanumeric': 0.08076877726683174, 'fraction_numerical': 0.0009432849899775969, 'mean_word_length': 2.786607142857143, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4002294', 'n_tokens_mistral': 2197, 'n_tokens_neox': 2126, 'n_words': 859}