<commit_before>from flask import g
import rethinkdb as r
from modules.model import Model
from models.mixins.entity import EntityMixin
from models.unit import Unit
from modules.validations import is_required, is_language, is_string, \
    is_boolean, is_list, is_entity_list_dict
from modules.util import uniqid


# TODO@ On set canonical, index (or delete) in Elasticsearch with entity_id
class Set(EntityMixin, Model):
    """
    A set is a collection of units and other sets.
    Sets can vary greatly in scale.
    A graph is automatically formed based on the units and sets specified.
    """
    tablename = 'sets'

    """
    The model represents a **version** of a set, not a set itself.
    The `entity_id` attribute is what refers to a particular set.
    The `id` attribute refers to a specific version of the set.
    The `previous_id` attribute refers to the version based off.
    """

    schema = dict(Model.schema.copy(), **{
        'entity_id': {
            'validate': (is_required, is_string,),  # TODO@ is valid id?
            'default': uniqid
        },
        'previous_id': {
            'validate': (is_string,),  # TODO@ is valid id?
        },
        'language': {
            'validate': (is_required, is_language,),
            'default': 'en'
        },
        'name': {
            'validate': (is_required, is_string,)
        },
        'body': {
            'validate': (is_required, is_string,)
        },
        'canonical': {
            'validate': (is_boolean,),
            'default': False
        },
        'available': {
            'validate': (is_boolean,),
            'default': True
        },
        'tags': {
            'validate': (is_list,),
            'default': []
        },
        'members': {
            'validate': (is_required, is_entity_list_dict,),
            # TODO@ is valid ids?
        }
    })

    def validate(self):
        errors = super().validate()
        if not errors:
            errors += self.ensure_no_cycles()
        return errors

    def ensure_no_cycles(self):
        """
        TODO@ Ensure no require cycles form.
        """
        return []

    @classmethod
    def list_by_unit_id(cls, unit_id):
        """
        Get a list of sets which contain the given member ID. Recursive.
        """

        # *** First, find the list of sets
        #     directly containing the member ID. ***

        query = (cls.start_canonicals_query()
                    .filter(r.row['members'].contains({
                        'kind': 'unit',
                        'id': unit_id,
                    })))
        sets = query.run(g.db_conn)

        # *** Second, find all the sets containing
        #     those sets... recursively. ***

        found_sets, all_sets = sets, []

        while found_sets:
            set_ids = set(set_['entity_id'] for set_ in found_sets)
            all_sets += found_sets
            query = (cls.start_canonicals_query()
                        .filter(r.row['members'].contains({
                            'kind': 'set',
                            'id': set_ids,  # TODO@
                        })))
            found_sets = query.run(g.db_conn)

        return all_sets

    def list_units(self):
        """
        Get the list of units contained within the set. Recursive. Connecting.
        """

        # *** First, we need to break down
        #     the set into a list of known units. ***

        unit_ids = set()
        sets = [self]

        while sets:
            set_ids = set()
            for set_ in sets:
                unit_ids.update(set(member['id']
                                    for member in getattr(set_, 'members', ())
                                    if member['kind'] == 'unit'))<commit_msg>Fixing queries in entity info routes<commit_after>from flask import g
import rethinkdb as r
from modules.model import Model
from models.mixins.entity import EntityMixin
from models.unit import Unit
from modules.validations import is_required, is_language, is_string, \
    is_boolean, is_list, is_entity_list_dict
from modules.util import uniqid


# TODO@ On set canonical, index (or delete) in Elasticsearch with entity_id
class Set(EntityMixin, Model):
    """
    A set is a collection of units and other sets.
    Sets can vary greatly in scale.
    A graph is automatically formed based on the units and sets specified.
    """
    tablename = 'sets'

    """
    The model represents a **version** of a set, not a set itself.
    The `entity_id` attribute is what refers to a particular set.
    The `id` attribute refers to a specific version of the set.
    The `previous_id` attribute refers to the version based off.
    """

    schema = dict(Model.schema.copy(), **{
        'entity_id': {
            'validate': (is_required, is_string,),  # TODO@ is valid id?
            'default': uniqid
        },
        'previous_id': {
            'validate': (is_string,),  # TODO@ is valid id?
        },
        'language': {
            'validate': (is_required, is_language,),
            'default': 'en'
        },
        'name': {
            'validate': (is_required, is_string,)
        },
        'body': {
            'validate': (is_required, is_string,)
        },
        'canonical': {
            'validate': (is_boolean,),
            'default': False
        },
        'available': {
            'validate': (is_boolean,),
            'default': True
        },
        'tags': {
            'validate': (is_list,),
            'default': []
        },
        'members': {
            'validate': (is_required, is_entity_list_dict,),
            # TODO@ is valid ids?
        }
    })

    def validate(self):
        errors = super().validate()
        if not errors:
            errors += self.ensure_no_cycles()
        return errors

    def ensure_no_cycles(self):
        """
        TODO@ Ensure no require cycles form.
        """
        return []

    @classmethod
    def list_by_unit_id(cls, unit_id):
        """
        Get a list of sets which contain the given member ID. Recursive.
        """

        # *** First, find the list of sets
        #     directly containing the member ID. ***

        """
        r.table('sets')
          .filter(r.row('members').contains(function(m) {
            return m('id').eq('B');
          }));
        """

        query = (cls.start_canonicals_query()
                    .filter(r.row['members'].contains(
                        lambda member:
                            member['id'] == unit_id
                    )))
        sets = query.run(g.db_conn)

        # *** Second, find all the sets containing
        #     those sets... recursively. ***

        found_sets, all_sets = sets, []

        """
        r.table('sets')
          .filter(r.row('members').contains(function(m) {
            return r.expr(['B']).contains(m('id'));
          }));
        """

        while found_sets:
            set_ids = set(set_['entity_id'] for set_ in found_sets)
            all_sets += found_sets
            query = (cls.start_canonicals_query()
                        .filter(r.row['members'].contains(
                            lambda member:
                                r.expr(set_ids).contains(member['id'])
                        )))
            found_sets = query.run(g.db_conn)

        return all_sets

    def list_units(self):
        """
        Get the list of units contained within the set. Recursive. Connecting.
        """

        # *** First, we need to break down
        #     the set into a list of known units. ***

        unit_ids = set()
        sets = [self]

        while sets:
            set_ids = set()
            for set_ in sets:
                unit_ids.update(set(member['id']
                                    for member in getattr(set_, 'members', ())
                                    if member['kind'] == 'unit'))
{'fraction_non_alphanumeric': 0.11560766515380737, 'fraction_numerical': 0.0, 'mean_word_length': 1.6164248021108178, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1827031', 'n_tokens_mistral': 2197, 'n_tokens_neox': 2150, 'n_words': 634}