<commit_before>from operator import attrgetter, itemgetter
from collections import defaultdict
from itertools import accumulate, repeat, chain, zip_longest

# For debugging only.
import sys
dbg = lambda *args: print(*args, file=sys.stderr)



#
# Total Ordering infrastructure
#


class RichCompareMixin:

    # Flexible rich compare adapted from recipe by <NAME>.

    def _compare(self, other, method):
        try:
            return method(self._cmpkey(), other._cmpkey())
        except TypeError:
            # _cmpkey returned an incommensurate type.
            return NotImplemented

    def __lt__(self, other):
        return self._compare(other, lambda s,o: s < o)

    def __le__(self, other):
        return self._compare(other, lambda s,o: s <= o)

    def __eq__(self, other):
        return self._compare(other, lambda s,o: s == o)

    def __ge__(self, other):
        return self._compare(other, lambda s,o: s >= o)

    def __gt__(self, other):
        return self._compare(other, lambda s,o: s > o)

    def __ne__(self, other):
        return self._compare(other, lambda s,o: s != o)

    def __hash__(self):
        return hash(self._cmpkey())



#
# User Defined type support
#


class Scaler(RichCompareMixin):

    def _cmpkey(self):
        return self.value

    def _compare(self, other, method):
        if type(self) != type(other):
            return NotImplemented
        return super()._compare(other, method)

    def __str__(self):
        return str(self.value)

    def __repr__(self):
        return "{}({!r})".format(self.__class__.__name__, self.value)



#
# Row types (TTM/Tutorial D TUPLE Types)
#


class RowMeta(type):

    def __eq__(self, other):
        if not isinstance(other, RowMeta):
            return NotImplemented
        return self._header_ == other._header_

    def __hash__(self):
        return super().__hash__()


class Row(RichCompareMixin, metaclass=RowMeta):

    def __init__(self, attrdict):
        if isinstance(attrdict, Row):
            # We are being called as a type function.
            if self._header_ != attrdict._header_:
                raise TypeError("Invalid Row type: {!r}".format(attrdict))
            self.__dict__ = attrdict.__dict__
            return
        if len(attrdict) != self._degree_:
            raise TypeError("Expected {} attributes, got {} ({!r})".format(
                                self._degree_, len(attrdict), attrdict))
        for attr, value in attrdict.items():
            try:
                setattr(self, attr, self._header_[attr](value))
            except (TypeError, ValueError) as e:
                raise type(e)(str(e) + "; {} invalid for attribute {}".format(
                                repr(value), attr))
            except KeyError:
                raise TypeError(
                    "Invalid attribute name {}".format(attr)) from None

    def __getitem__(self, key):
        return getattr(self, key)

    def __len__(self):
        return len(self.__dict__)

    def __iter__(self):
        return iter(self.__dict__)

    def _cmpkey(self):
        return tuple(sorted(self.__dict__.items()))

    def _compare(self, other, method):
        if not isinstance(other, Row) or self._header_ != other._header_:
            return NotImplemented
        return super()._compare(other, method)

    def __eq__(self, other):
        if not isinstance(other, Row):
            return False
        return self._cmpkey() == other._cmpkey()

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash(self._cmpkey())

    def __repr__(self):
        return "row({{{}}})".format(
            ', '.join("{!r}: {!r}".format(k, v)
                        for k, v in sorted(self.__dict__.items())))

    def __str__(self):
        return '({})'.format(
            ', '.join('{}={}'.format(k, v)
                        for k, v in sorted(self.__dict__.items())))

    def _as_dict_(self):
        return self.__dict__.copy()

    def _as_locals_(self):
        l = self.__dict__.copy()
        l['r'] = self
        return l



#
# Relation types
#


class RelationMeta(type):

    def __new__(cls, name, bases, dct):
        attrs = [x for x in dct if not x.startswith('_')]
        header = {name: dct.pop(name) for name in attrs}
        dct['header'] = header
        dct['degree'] = len(attrs)
        class RowClass(Row):
            _header_ = header
            _degree_ = len(attrs)
        dct['row'] = RowClass
        RowClass.__name__ = '.'.join((name, 'RowClass'))
        return type.__new__(cls, name, bases, dct)

    # XXX: This doesn't work, punt on it for now.
    #@property
    #def header(self):
    #    return self._header

    #@property
    #def degree(self):
    #    return len(self._header)

    def __eq__(self, other):
        if not isinstance(other, RelationMeta):
            return NotImplemented
        return self.header == other.header

    def __hash__(self):
        return super().__hash__()

    def __repr__(self):
        return "rel({{{}}})".format(
            ', '.join([repr(n)+': '+(v.__name__)
                       for n, v in sorted(self.header.items())]))


class Relation(RichCompareMixin, metaclass=RelationMeta):

    def __init__(self, *args):
        # Several cases: (1) empty relation (2) being called as a type
        # validation function (single arg is a Relation) (3) header tuple
        # followed by value tuples (4) iterable of dict-likes and/or Rows,
        # which could be a single argument or could be one dict/Row per arg.
        if len(args) == 0:
            # (1) Empty relation.
            self._rows_ = set()
            return
        if (len(args)==1 and isinstance(args[0], Relation) and
                args[0].header == self.header):
            # (2) We were called as a type validation function.  Return an
            # immutable copy, because the only time this happens is when a
            # relation is the value of an attribute, and when a relation is a
            # value of an attribute it must be immutable.
            self._rows_ = frozenset(args[0]._rows_)
            return
        rows = []
        first = None
        if (not hasattr(args[0], 'items') and
                not hasattr(args[0], 'union') and
                not hasattr(args[0], '_header_') and
                hasattr(args[0], '__iter__') and
                hasattr(args[0], '__len__') and
                isinstance(next(iter(args[0])), str)):
            # (3) Tuple form.  Tuple form is a special shorthand, and we only
            # allow it in argument list form, not single-iterator-argument
            # form.  Furthermore, while the list of arguments could be sourced
            # from a generator (not that that would be a good idea), the
            # arguments themselves cannot be.
            attrlist = args[0]
            self._validate_attr_list(attrlist)
            for i, o in enumerate(args[1:], start=1):
                if len(o) != self.degree:
                    raise TypeError( "Expected {} attributes, got {} in row {} "
                        "for relation type {}".format(
                            self.degree, len(o), i, repr(self.__class__)))
                try:
                    rows.append(self.row({k: v for k, v in zip(attrlist, o)}))
                except TypeError as e:
                    raise TypeError(str(e) + " in row {}".format(i))
        else:
            # (4) iterator of dicts and/or Rows.
            if len(args) == 1 and not (hasattr(args[0], 'items') or
                                       hasattr(args[0], '_header_')):
                # Single iterator argument form.
                args = args[0]
            for i, o in enumerate(args):
                if hasattr(o, '_header_'):
                    # This one is a row.
                    if o._header_ != self.header:
                        raise TypeError("Row header does not match relation header "
                                        "in row {} (got {!r} for relation type "
                                        "{!r})".format(i, o, type(self)))
                else:
                    # This one is a dict, turn it into a row.
                    try:
                        o = self.row(o)
                    except (ValueError, TypeError) as e:
                        raise type(e)(str(e) + " in row {}".format(i))
                rows.append(o)
        # One way or another we now have a list of Row objects.
        rowset = set()
        for i, r in enumerate(rows):
            if r in rowset:
                raise ValueError(
                    "Duplicate row: {!r} in row {} of input".format(r, i))
            rowset.add(r)
        self._rows_ = rowset

    def _validate_attr_list(self, attrlist):
        if len(attrlist) != self.degree:
            raise TypeError("Expected {} attributes, got {} ({}) in "
                    "header row for relation type {}".format(
                    self.degree,
                    len(attrlist), sorted(attrlist),
                    repr(self.__class__)))
        for attr in attrlist:
            if attr not in self.header:
                raise AttributeError(
                    "Relation type {!r} has no attribute {!r}".format(
                         repr(self.__class__), attr))

    # Miscellaneous operators.

    def __iter__(self):
        return iter(self._rows_)

    def __len__(self):
        return len(self._rows_)

    # Comparison operators (see RichCompareMixin).

    def _cmpkey(self):
        return self._rows_

    def _compare(self, other, method):
        if not isinstance(other, Relation) or self.header != other.header:
            return NotImplemented
        return super()._compare(other, method)

    def __eq__(self, other):
        if not isinstance(other, Relation):
            return False
        if self.header != other.header:
            return False
        return self._rows_ == other._rows_

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash(self._rows_)

    # Infix relational operators.

    def __and__(self, other):                   # &
        return _binary_join(self, other)

    def __rshift__(self, other):                # >>
        return project(self, other)

    def __lshift__(self, other):                # <<
        return project(self, all_but(other))

    def __or__(self, other):                    # |
        return union(self, other)

    def __sub__(self, other):                   # -
        return notmatching(self, other)

    # Presentation operators.

    def __repr__(self):
        names = sorted(self.header)
        if self._rows_:
            return "rel({{{}}})".format(
                ', '.join(repr(row)
                          for row in sorted(self._rows_)))
        return repr(self.__class__) + '()'

    def __str__(self):
        return _display(self, *sorted(self.header))


class DumDee(Relation):
    pass

Dum = DumDee()
Dee = DumDee({})



#
# Row and Relation creation.
#


def row(*args, **kw):
    if len(args)==1:
        kw = dict(args[0], **kw)
    if any(n.startswith('_') for n in kw):
        raise ValueError("Invalid relational attribute name {!r}".format(
            [n for n in sorted(kw) if n.startswith('_')][0]))
    dct = {'_header_': {n: type(v) for n, v in kw.items()},
           '_degree_': len(kw)}
    cls = type('row_' + '_'.join(sorted(kw.keys())), (Row,), dct)
    return cls(kw)


def rel(*args, **kw):
    # Three possibilities.  (1) no arguments or keywords, or a single argument
    # that is an empty dictionary or set: a relation literal representing Dum.
    # (2) a single dictionary argument whose values are all types, possibly
    # with additional keyword arguments whose values are types: a relation<commit_msg>Eliminate DumDee class.<commit_after>from operator import attrgetter, itemgetter
from collections import defaultdict
from itertools import accumulate, repeat, chain, zip_longest

# For debugging only.
import sys
dbg = lambda *args: print(*args, file=sys.stderr)



#
# Total Ordering infrastructure
#


class RichCompareMixin:

    # Flexible rich compare adapted from recipe by <NAME>.

    def _compare(self, other, method):
        try:
            return method(self._cmpkey(), other._cmpkey())
        except TypeError:
            # _cmpkey returned an incommensurate type.
            return NotImplemented

    def __lt__(self, other):
        return self._compare(other, lambda s,o: s < o)

    def __le__(self, other):
        return self._compare(other, lambda s,o: s <= o)

    def __eq__(self, other):
        return self._compare(other, lambda s,o: s == o)

    def __ge__(self, other):
        return self._compare(other, lambda s,o: s >= o)

    def __gt__(self, other):
        return self._compare(other, lambda s,o: s > o)

    def __ne__(self, other):
        return self._compare(other, lambda s,o: s != o)

    def __hash__(self):
        return hash(self._cmpkey())



#
# User Defined type support
#


class Scaler(RichCompareMixin):

    def _cmpkey(self):
        return self.value

    def _compare(self, other, method):
        if type(self) != type(other):
            return NotImplemented
        return super()._compare(other, method)

    def __str__(self):
        return str(self.value)

    def __repr__(self):
        return "{}({!r})".format(self.__class__.__name__, self.value)



#
# Row types (TTM/Tutorial D TUPLE Types)
#


class RowMeta(type):

    def __eq__(self, other):
        if not isinstance(other, RowMeta):
            return NotImplemented
        return self._header_ == other._header_

    def __hash__(self):
        return super().__hash__()


class Row(RichCompareMixin, metaclass=RowMeta):

    def __init__(self, attrdict):
        if isinstance(attrdict, Row):
            # We are being called as a type function.
            if self._header_ != attrdict._header_:
                raise TypeError("Invalid Row type: {!r}".format(attrdict))
            self.__dict__ = attrdict.__dict__
            return
        if len(attrdict) != self._degree_:
            raise TypeError("Expected {} attributes, got {} ({!r})".format(
                                self._degree_, len(attrdict), attrdict))
        for attr, value in attrdict.items():
            try:
                setattr(self, attr, self._header_[attr](value))
            except (TypeError, ValueError) as e:
                raise type(e)(str(e) + "; {} invalid for attribute {}".format(
                                repr(value), attr))
            except KeyError:
                raise TypeError(
                    "Invalid attribute name {}".format(attr)) from None

    def __getitem__(self, key):
        return getattr(self, key)

    def __len__(self):
        return len(self.__dict__)

    def __iter__(self):
        return iter(self.__dict__)

    def _cmpkey(self):
        return tuple(sorted(self.__dict__.items()))

    def _compare(self, other, method):
        if not isinstance(other, Row) or self._header_ != other._header_:
            return NotImplemented
        return super()._compare(other, method)

    def __eq__(self, other):
        if not isinstance(other, Row):
            return False
        return self._cmpkey() == other._cmpkey()

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash(self._cmpkey())

    def __repr__(self):
        return "row({{{}}})".format(
            ', '.join("{!r}: {!r}".format(k, v)
                        for k, v in sorted(self.__dict__.items())))

    def __str__(self):
        return '({})'.format(
            ', '.join('{}={}'.format(k, v)
                        for k, v in sorted(self.__dict__.items())))

    def _as_dict_(self):
        return self.__dict__.copy()

    def _as_locals_(self):
        l = self.__dict__.copy()
        l['r'] = self
        return l



#
# Relation types
#


class RelationMeta(type):

    def __new__(cls, name, bases, dct):
        attrs = [x for x in dct if not x.startswith('_')]
        header = {name: dct.pop(name) for name in attrs}
        dct['header'] = header
        dct['degree'] = len(attrs)
        class RowClass(Row):
            _header_ = header
            _degree_ = len(attrs)
        dct['row'] = RowClass
        RowClass.__name__ = '.'.join((name, 'RowClass'))
        return type.__new__(cls, name, bases, dct)

    # XXX: This doesn't work, punt on it for now.
    #@property
    #def header(self):
    #    return self._header

    #@property
    #def degree(self):
    #    return len(self._header)

    def __eq__(self, other):
        if not isinstance(other, RelationMeta):
            return NotImplemented
        return self.header == other.header

    def __hash__(self):
        return super().__hash__()

    def __repr__(self):
        return "rel({{{}}})".format(
            ', '.join([repr(n)+': '+(v.__name__)
                       for n, v in sorted(self.header.items())]))


class Relation(RichCompareMixin, metaclass=RelationMeta):

    def __init__(self, *args):
        # Several cases: (1) empty relation (2) being called as a type
        # validation function (single arg is a Relation) (3) header tuple
        # followed by value tuples (4) iterable of dict-likes and/or Rows,
        # which could be a single argument or could be one dict/Row per arg.
        if len(args) == 0:
            # (1) Empty relation.
            self._rows_ = set()
            return
        if (len(args)==1 and isinstance(args[0], Relation) and
                args[0].header == self.header):
            # (2) We were called as a type validation function.  Return an
            # immutable copy, because the only time this happens is when a
            # relation is the value of an attribute, and when a relation is a
            # value of an attribute it must be immutable.
            self._rows_ = frozenset(args[0]._rows_)
            return
        rows = []
        first = None
        if (not hasattr(args[0], 'items') and
                not hasattr(args[0], 'union') and
                not hasattr(args[0], '_header_') and
                hasattr(args[0], '__iter__') and
                hasattr(args[0], '__len__') and
                isinstance(next(iter(args[0])), str)):
            # (3) Tuple form.  Tuple form is a special shorthand, and we only
            # allow it in argument list form, not single-iterator-argument
            # form.  Furthermore, while the list of arguments could be sourced
            # from a generator (not that that would be a good idea), the
            # arguments themselves cannot be.
            attrlist = args[0]
            self._validate_attr_list(attrlist)
            for i, o in enumerate(args[1:], start=1):
                if len(o) != self.degree:
                    raise TypeError( "Expected {} attributes, got {} in row {} "
                        "for relation type {}".format(
                            self.degree, len(o), i, repr(self.__class__)))
                try:
                    rows.append(self.row({k: v for k, v in zip(attrlist, o)}))
                except TypeError as e:
                    raise TypeError(str(e) + " in row {}".format(i))
        else:
            # (4) iterator of dicts and/or Rows.
            if len(args) == 1 and not (hasattr(args[0], 'items') or
                                       hasattr(args[0], '_header_')):
                # Single iterator argument form.
                args = args[0]
            for i, o in enumerate(args):
                if hasattr(o, '_header_'):
                    # This one is a row.
                    if o._header_ != self.header:
                        raise TypeError("Row header does not match relation header "
                                        "in row {} (got {!r} for relation type "
                                        "{!r})".format(i, o, type(self)))
                else:
                    # This one is a dict, turn it into a row.
                    try:
                        o = self.row(o)
                    except (ValueError, TypeError) as e:
                        raise type(e)(str(e) + " in row {}".format(i))
                rows.append(o)
        # One way or another we now have a list of Row objects.
        rowset = set()
        for i, r in enumerate(rows):
            if r in rowset:
                raise ValueError(
                    "Duplicate row: {!r} in row {} of input".format(r, i))
            rowset.add(r)
        self._rows_ = rowset

    def _validate_attr_list(self, attrlist):
        if len(attrlist) != self.degree:
            raise TypeError("Expected {} attributes, got {} ({}) in "
                    "header row for relation type {}".format(
                    self.degree,
                    len(attrlist), sorted(attrlist),
                    repr(self.__class__)))
        for attr in attrlist:
            if attr not in self.header:
                raise AttributeError(
                    "Relation type {!r} has no attribute {!r}".format(
                         repr(self.__class__), attr))

    # Miscellaneous operators.

    def __iter__(self):
        return iter(self._rows_)

    def __len__(self):
        return len(self._rows_)

    # Comparison operators (see RichCompareMixin).

    def _cmpkey(self):
        return self._rows_

    def _compare(self, other, method):
        if not isinstance(other, Relation) or self.header != other.header:
            return NotImplemented
        return super()._compare(other, method)

    def __eq__(self, other):
        if not isinstance(other, Relation):
            return False
        if self.header != other.header:
            return False
        return self._rows_ == other._rows_

    def __ne__(self, other):
        return not self == other

    def __hash__(self):
        return hash(self._rows_)

    # Infix relational operators.

    def __and__(self, other):                   # &
        return _binary_join(self, other)

    def __rshift__(self, other):                # >>
        return project(self, other)

    def __lshift__(self, other):                # <<
        return project(self, all_but(other))

    def __or__(self, other):                    # |
        return union(self, other)

    def __sub__(self, other):                   # -
        return notmatching(self, other)

    # Presentation operators.

    def __repr__(self):
        names = sorted(self.header)
        if self._rows_:
            return "rel({{{}}})".format(
                ', '.join(repr(row)
                          for row in sorted(self._rows_)))
        return repr(self.__class__) + '()'

    def __str__(self):
        return _display(self, *sorted(self.header))


Dum = Relation()
Dee = Relation({})



#
# Row and Relation creation.
#


def row(*args, **kw):
    if len(args)==1:
        kw = dict(args[0], **kw)
    if any(n.startswith('_') for n in kw):
        raise ValueError("Invalid relational attribute name {!r}".format(
            [n for n in sorted(kw) if n.startswith('_')][0]))
    dct = {'_header_': {n: type(v) for n, v in kw.items()},
           '_degree_': len(kw)}
    cls = type('row_' + '_'.join(sorted(kw.keys())), (Row,), dct)
    return cls(kw)


def rel(*args, **kw):
    # Three possibilities.  (1) no arguments or keywords, or a single argument
    # that is an empty dictionary or set: a relation literal representing Dum.
    # (2) a single dictionary argument whose values are all types, possibly
    # with additional keyword arguments whose values are types: a relation
{'fraction_non_alphanumeric': 0.11498904063395718, 'fraction_numerical': 0.00261338728713539, 'mean_word_length': 1.8660304421357816, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6220590', 'n_tokens_mistral': 6862, 'n_tokens_neox': 6581, 'n_words': 2060}