<commit_before>"""
classes for relational algebra
"""

import numpy as np
import abc
import re
from copy import copy
from datajoint import DataJointError
from .blob import unpack
import logging
import numpy.lib.recfunctions as rfn

logger = logging.getLogger(__name__)


class RelationalOperand(metaclass=abc.ABCMeta):
    """
    RelationalOperand implements relational algebra and fetch methods.
    RelationalOperand objects reference other relation objects linked by operators.
    The leaves of this tree of objects are base relations.
    When fetching data from the database, this tree of objects is compiled into an SQL expression.
    It is a mixin class that provides relational operators, iteration, and fetch capability.
    RelationalOperand operators are: restrict, pro, and join.
    """

    def __init__(self, conn, restrictions=None):
        self._conn = conn
        self._restrictions = [] if restrictions is None else restrictions

    @property
    def conn(self):
        return self._conn

    @property
    def restrictions(self):
        return self._restrictions

    @abc.abstractproperty
    def from_clause(self):
        pass

    @abc.abstractproperty
    def heading(self):
        pass

    def __mul__(self, other):
        """
        relational join
        """
        return Join(self, other)

    def __mod__(self, attributes=None):
        """
        relational projection operator.  See RelationalOperand.project
        """
        return self.project(*attributes)

    def project(self, *attributes, **renamed_attributes):
        """
        Relational projection operator.
        :param attributes: a list of attribute names to be included in the result.
        :return: a new relation with selected fields
        Primary key attributes are always selected and cannot be excluded.
        Therefore obj.project() produces a relation with only the primary key attributes.
        If attributes includes the string '*', all attributes are selected.
        Each attribute can only be used once in attributes or renamed_attributes.  Therefore, the projected
        relation cannot have more attributes than the original relation.
        """
        # if the first attribute is a relation, it will be aggregated
        group = None
        if attributes and isinstance(attributes[0], RelationalOperand):
            group = attributes.pop[0]
        return Projection(self, group, *attributes, **renamed_attributes)

    def aggregate(self, _group, *attributes, **renamed_attributes):
        """
        Relational aggregation operator
        :param group:  relation whose tuples can be used in aggregation operators
        :param extensions:
        :return: a relation representing the aggregation/projection operator result
        """
        if _group is not None and not isinstance(_group, RelationalOperand):
            raise DataJointError('The second argument must be a relation or None')
        return Projection(self, _group, *attributes, **renamed_attributes)

    def __iand__(self, restriction):
        """
        in-place relational restriction or semijoin
        """
        if restriction is not None:
            if self._restrictions is None:
                self._restrictions = []
            self._restrictions.append(restriction)
        return self

    def __and__(self, restriction):
        """
        relational restriction or semijoin
        """
        if self._restrictions is None:
            self._restrictions = []
        ret = copy(self)  # todo: why not deepcopy it?
        ret._restrictions = list(ret._restrictions)  # copy restriction
        ret &= restriction
        return ret

    def __isub__(self, restriction):
        """
        in-place antijoin (inverted restriction)
        """
        self &= Not(restriction)
        return self

    def __sub__(self, restriction):
        """
        inverted restriction aka antijoin
        """
        return self & Not(restriction)

    def make_select(self, attribute_spec=None):
        if attribute_spec is None:
            attribute_spec = self.heading.as_sql
        return 'SELECT ' + attribute_spec + ' FROM ' + self.from_clause + self.where_clause

    def __len__(self):
        """
        number of tuples in the relation.  This also takes care of the truth value
        """
        cur = self.conn.query(self.make_select('count(*)'))
        return cur.fetchone()[0]

    def __contains__(self, item):
        """
        "item in relation" is equivalient to "len(relation & item)>0"
        """
        return len(self & item)>0

    def __call__(self, *args, **kwargs):
        """
        calling a relation is equivalent to fetching from it
        """
        return self.fetch(*args, **kwargs)

    def fetch(self, offset=0, limit=None, order_by=None, descending=False, as_dict=False):
        """
        fetches the relation from the database table into an np.array and unpacks blob attributes.
        :param offset: the number of tuples to skip in the returned result
        :param limit: the maximum number of tuples to return
        :param order_by: the list of attributes to order the results. No ordering should be assumed if order_by=None.
        :param descending: the list of attributes to order the results
        :param as_dict: returns a list of dictionaries instead of a record array
        :return: the contents of the relation in the form of a structured numpy.array
        """
        cur = self.cursor(offset, limit, order_by, descending)
        if as_dict:
            attr_names = self.heading.names
            ret = [dict((n, unpack(e)) if n in self.heading.blobs else (n, e)
                        for n, e in zip(attr_names, t)) for t in cur.fetchall()]
        else:
            ret = np.array(list(cur.fetchall()), dtype=self.heading.as_dtype)
            for blob_name in self.heading.blobs:
                ret[blob_name] = list(map(unpack, ret[blob_name]))
        return ret

    def cursor(self, offset=0, limit=None, order_by=None, descending=False):
        """
        Return query cursor.
        See Relation.fetch() for input description.
        :return: cursor to the query
        """
        if offset and limit is None:
            raise DataJointError('limit is required when offset is set')
        sql = self.make_select()
        if order_by is not None:
            sql += ' ORDER BY ' + ', '.join(order_by)
            if descending:
                sql += ' DESC'
        if limit is not None:
            sql += ' LIMIT %d' % limit
            if offset:
                sql += ' OFFSET %d' % offset
        logger.debug(sql)
        return self.conn.query(sql)

    def __repr__(self):
        limit = 7 #TODO: move some of these display settings into the config
        width = 14
        rel = self.project(*self.heading.non_blobs)
        template = '%%-%d.%ds' % (width, width)
        columns = rel.heading.names
        repr_string = ' '.join([template % column for column in columns]) + '\n'
        repr_string += ' '.join(['+' + '-'*(width-2) + '+' for _ in columns]) + '\n'
        for tup in rel.fetch(limit=limit):
            repr_string += ' '.join([template % column for column in tup]) + '\n'
        if self.count > limit:
            repr_string += '...\n'
        repr_string += ' (%d tuples)\n' % len(self)
        return repr_string
        
    def __iter__(self):
        """
        Iterator that yields individual tuples of the current table dictionaries.
        """
        cur = self.cursor()
        heading = self.heading  # construct once for efficiency
        do_unpack = tuple(h in heading.blobs for h in self.heading.names)
        values = cur.fetchone()
        while values:
            yield {field_name: unpack(value) if up else value
                   for field_name, up, value in zip(heading.names, do_unpack, values)}
            values = cur.fetchone()

    @property
    def where_clause(self):
        """
        convert the restriction into an SQL WHERE
        """
        if not self.restrictions:
            return ''
        
        def make_condition(arg):
            if isinstance(arg, dict):
                conditions = ['`%s`=%s' % (k, repr(v)) for k, v in arg.items()]
            elif isinstance(arg, np.void):<commit_msg>bug fix: replaced rel.count with len(rel)<commit_after>"""
classes for relational algebra
"""

import numpy as np
import abc
import re
from copy import copy
from datajoint import DataJointError
from .blob import unpack
import logging
import numpy.lib.recfunctions as rfn

logger = logging.getLogger(__name__)


class RelationalOperand(metaclass=abc.ABCMeta):
    """
    RelationalOperand implements relational algebra and fetch methods.
    RelationalOperand objects reference other relation objects linked by operators.
    The leaves of this tree of objects are base relations.
    When fetching data from the database, this tree of objects is compiled into an SQL expression.
    It is a mixin class that provides relational operators, iteration, and fetch capability.
    RelationalOperand operators are: restrict, pro, and join.
    """

    def __init__(self, conn, restrictions=None):
        self._conn = conn
        self._restrictions = [] if restrictions is None else restrictions

    @property
    def conn(self):
        return self._conn

    @property
    def restrictions(self):
        return self._restrictions

    @abc.abstractproperty
    def from_clause(self):
        pass

    @abc.abstractproperty
    def heading(self):
        pass

    def __mul__(self, other):
        """
        relational join
        """
        return Join(self, other)

    def __mod__(self, attributes=None):
        """
        relational projection operator.  See RelationalOperand.project
        """
        return self.project(*attributes)

    def project(self, *attributes, **renamed_attributes):
        """
        Relational projection operator.
        :param attributes: a list of attribute names to be included in the result.
        :return: a new relation with selected fields
        Primary key attributes are always selected and cannot be excluded.
        Therefore obj.project() produces a relation with only the primary key attributes.
        If attributes includes the string '*', all attributes are selected.
        Each attribute can only be used once in attributes or renamed_attributes.  Therefore, the projected
        relation cannot have more attributes than the original relation.
        """
        # if the first attribute is a relation, it will be aggregated
        group = None
        if attributes and isinstance(attributes[0], RelationalOperand):
            group = attributes.pop[0]
        return Projection(self, group, *attributes, **renamed_attributes)

    def aggregate(self, _group, *attributes, **renamed_attributes):
        """
        Relational aggregation operator
        :param group:  relation whose tuples can be used in aggregation operators
        :param extensions:
        :return: a relation representing the aggregation/projection operator result
        """
        if _group is not None and not isinstance(_group, RelationalOperand):
            raise DataJointError('The second argument must be a relation or None')
        return Projection(self, _group, *attributes, **renamed_attributes)

    def __iand__(self, restriction):
        """
        in-place relational restriction or semijoin
        """
        if restriction is not None:
            if self._restrictions is None:
                self._restrictions = []
            self._restrictions.append(restriction)
        return self

    def __and__(self, restriction):
        """
        relational restriction or semijoin
        """
        if self._restrictions is None:
            self._restrictions = []
        ret = copy(self)  # todo: why not deepcopy it?
        ret._restrictions = list(ret._restrictions)  # copy restriction
        ret &= restriction
        return ret

    def __isub__(self, restriction):
        """
        in-place antijoin (inverted restriction)
        """
        self &= Not(restriction)
        return self

    def __sub__(self, restriction):
        """
        inverted restriction aka antijoin
        """
        return self & Not(restriction)

    def make_select(self, attribute_spec=None):
        if attribute_spec is None:
            attribute_spec = self.heading.as_sql
        return 'SELECT ' + attribute_spec + ' FROM ' + self.from_clause + self.where_clause

    def __len__(self):
        """
        number of tuples in the relation.  This also takes care of the truth value
        """
        cur = self.conn.query(self.make_select('count(*)'))
        return cur.fetchone()[0]

    def __contains__(self, item):
        """
        "item in relation" is equivalient to "len(relation & item)>0"
        """
        return len(self & item)>0

    def __call__(self, *args, **kwargs):
        """
        calling a relation is equivalent to fetching from it
        """
        return self.fetch(*args, **kwargs)

    def fetch(self, offset=0, limit=None, order_by=None, descending=False, as_dict=False):
        """
        fetches the relation from the database table into an np.array and unpacks blob attributes.
        :param offset: the number of tuples to skip in the returned result
        :param limit: the maximum number of tuples to return
        :param order_by: the list of attributes to order the results. No ordering should be assumed if order_by=None.
        :param descending: the list of attributes to order the results
        :param as_dict: returns a list of dictionaries instead of a record array
        :return: the contents of the relation in the form of a structured numpy.array
        """
        cur = self.cursor(offset, limit, order_by, descending)
        if as_dict:
            attr_names = self.heading.names
            ret = [dict((n, unpack(e)) if n in self.heading.blobs else (n, e)
                        for n, e in zip(attr_names, t)) for t in cur.fetchall()]
        else:
            ret = np.array(list(cur.fetchall()), dtype=self.heading.as_dtype)
            for blob_name in self.heading.blobs:
                ret[blob_name] = list(map(unpack, ret[blob_name]))
        return ret

    def cursor(self, offset=0, limit=None, order_by=None, descending=False):
        """
        Return query cursor.
        See Relation.fetch() for input description.
        :return: cursor to the query
        """
        if offset and limit is None:
            raise DataJointError('limit is required when offset is set')
        sql = self.make_select()
        if order_by is not None:
            sql += ' ORDER BY ' + ', '.join(order_by)
            if descending:
                sql += ' DESC'
        if limit is not None:
            sql += ' LIMIT %d' % limit
            if offset:
                sql += ' OFFSET %d' % offset
        logger.debug(sql)
        return self.conn.query(sql)

    def __repr__(self):
        limit = 7 #TODO: move some of these display settings into the config
        width = 14
        rel = self.project(*self.heading.non_blobs)
        template = '%%-%d.%ds' % (width, width)
        columns = rel.heading.names
        repr_string = ' '.join([template % column for column in columns]) + '\n'
        repr_string += ' '.join(['+' + '-'*(width-2) + '+' for _ in columns]) + '\n'
        for tup in rel.fetch(limit=limit):
            repr_string += ' '.join([template % column for column in tup]) + '\n'
        if len(self) > limit:
            repr_string += '...\n'
        repr_string += ' (%d tuples)\n' % len(self)
        return repr_string
        
    def __iter__(self):
        """
        Iterator that yields individual tuples of the current table dictionaries.
        """
        cur = self.cursor()
        heading = self.heading  # construct once for efficiency
        do_unpack = tuple(h in heading.blobs for h in self.heading.names)
        values = cur.fetchone()
        while values:
            yield {field_name: unpack(value) if up else value
                   for field_name, up, value in zip(heading.names, do_unpack, values)}
            values = cur.fetchone()

    @property
    def where_clause(self):
        """
        convert the restriction into an SQL WHERE
        """
        if not self.restrictions:
            return ''
        
        def make_condition(arg):
            if isinstance(arg, dict):
                conditions = ['`%s`=%s' % (k, repr(v)) for k, v in arg.items()]
            elif isinstance(arg, np.void):
{'fraction_non_alphanumeric': 0.08847699598104493, 'fraction_numerical': 0.0013196568892088058, 'mean_word_length': 2.262622309197652, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3604638', 'n_tokens_mistral': 4387, 'n_tokens_neox': 4283, 'n_words': 1572}