<commit_before>"""autopopulate containing the dj.AutoPopulate class. See `dj.AutoPopulate` for more info."""
import logging
import datetime
import random
from tqdm import tqdm
from itertools import count
from pymysql import OperationalError
from .relational_operand import RelationalOperand, AndList, U
from . import DataJointError
from . import key as KEY
from .base_relation import FreeRelation
import signal

# noinspection PyExceptionInherit,PyCallingNonCallable

logger = logging.getLogger(__name__)


class AutoPopulate:
    """
    AutoPopulate is a mixin class that adds the method populate() to a Relation class.
    Auto-populated relations must inherit from both Relation and AutoPopulate,
    must define the property `key_source`, and must define the callback method _make_tuples.
    """
    _key_source = None

    @property
    def key_source(self):
        """
        :return: the relation whose primary key values are passed, sequentially, to the
                `_make_tuples` method when populate() is called.The default value is the
                join of the parent relations. Users may override to change the granularity
                or the scope of populate() calls.
        """
        if self._key_source is None:
            self.connection.dependencies.load(self.full_table_name)
            parents = list(self.target.parents(primary=True))
            if not parents:
                raise DataJointError('A relation must have parent relations to be able to be populated')
            self._key_source = FreeRelation(self.connection, parents.pop(0)).proj()
            while parents:
                self._key_source *= FreeRelation(self.connection, parents.pop(0)).proj()
        return self._key_source

    def _make_tuples(self, key):
        """
        Derived classes must implement method _make_tuples that fetches data from tables that are
        above them in the dependency hierarchy, restricting by the given key, computes dependent
        attributes, and inserts the new tuples into self.
        """
        raise NotImplementedError('Subclasses of AutoPopulate must implement the method "_make_tuples"')

    @property
    def target(self):
        """
        relation to be populated.
        Typically, AutoPopulate are mixed into a Relation object and the target is self.
        """
        return self

    def _job_key(self, key):
        """
        :param key:  they key returned for the job from the key source
        :return: the dict to use to generate the job reservation hash
        """
        return key

    def populate(self, *restrictions, suppress_errors=False, reserve_jobs=False,
		 order="original", limit=None, max_calls=None, display_progress=False):
        """
        rel.populate() calls rel._make_tuples(key) for every primary key in self.key_source
        for which there is not already a tuple in rel.

        :param restrictions: a list of restrictions each restrict (rel.key_source - target.proj())
        :param suppress_errors: suppresses error if true
        :param reserve_jobs: if true, reserves job to populate in asynchronous fashion
        :param order: "original"|"reverse"|"random"  - the order of execution
        :param display_progress: if True, report progress_bar
        :param limit: if not None, checks at most that many keys
        :param max_calls: if not None, populates at max that many keys
        """
        if self.connection.in_transaction:
            raise DataJointError('Populate cannot be called during a transaction.')

        valid_order = ['original', 'reverse', 'random']
        if order not in valid_order:
            raise DataJointError('The order argument must be one of %s' % str(valid_order))

        todo = self.key_source
        if not isinstance(todo, RelationalOperand):
            raise DataJointError('Invalid key_source value')
        todo = (todo & AndList(restrictions)).proj()
        if any(name not in self.target.heading for name in todo.heading):
            raise DataJointError('The populated target must have all the attributes of the key source')
        todo -= self.target

        error_list = [] if suppress_errors else None
        jobs = self.connection.schemas[self.target.database].jobs if reserve_jobs else None

        # define and setup signal handler for SIGTERM
        if reserve_jobs:
            def handler(signum, frame):
                logger.info('Populate terminated by SIGTERM')
                raise SystemExit('SIGTERM received')
            old_handler = signal.signal(signal.SIGTERM, handler)

        keys = todo.fetch(KEY, limit=limit)
        if order == "reverse":
            keys.reverse()
        elif order == "random":
            random.shuffle(keys)

        call_count = count()
        logger.info('Found %d keys to populate' % len(keys))

        for key in (tqdm(keys) if display_progress else keys):
            if max_calls is not None and call_count >= max_calls:
                break
            if not reserve_jobs or jobs.reserve(self.target.table_name, self._job_key(key)):
                self.connection.start_transaction()<commit_msg>improved the error message in autopopulate<commit_after>"""autopopulate containing the dj.AutoPopulate class. See `dj.AutoPopulate` for more info."""
import logging
import datetime
import random
from tqdm import tqdm
from itertools import count
from pymysql import OperationalError
from .relational_operand import RelationalOperand, AndList, U
from . import DataJointError
from . import key as KEY
from .base_relation import FreeRelation
import signal

# noinspection PyExceptionInherit,PyCallingNonCallable

logger = logging.getLogger(__name__)


class AutoPopulate:
    """
    AutoPopulate is a mixin class that adds the method populate() to a Relation class.
    Auto-populated relations must inherit from both Relation and AutoPopulate,
    must define the property `key_source`, and must define the callback method _make_tuples.
    """
    _key_source = None

    @property
    def key_source(self):
        """
        :return: the relation whose primary key values are passed, sequentially, to the
                `_make_tuples` method when populate() is called.The default value is the
                join of the parent relations. Users may override to change the granularity
                or the scope of populate() calls.
        """
        if self._key_source is None:
            self.connection.dependencies.load(self.full_table_name)
            parents = list(self.target.parents(primary=True))
            if not parents:
                raise DataJointError('A relation must have parent relations to be able to be populated')
            self._key_source = FreeRelation(self.connection, parents.pop(0)).proj()
            while parents:
                self._key_source *= FreeRelation(self.connection, parents.pop(0)).proj()
        return self._key_source

    def _make_tuples(self, key):
        """
        Derived classes must implement method _make_tuples that fetches data from tables that are
        above them in the dependency hierarchy, restricting by the given key, computes dependent
        attributes, and inserts the new tuples into self.
        """
        raise NotImplementedError('Subclasses of AutoPopulate must implement the method "_make_tuples"')

    @property
    def target(self):
        """
        relation to be populated.
        Typically, AutoPopulate are mixed into a Relation object and the target is self.
        """
        return self

    def _job_key(self, key):
        """
        :param key:  they key returned for the job from the key source
        :return: the dict to use to generate the job reservation hash
        """
        return key

    def populate(self, *restrictions, suppress_errors=False, reserve_jobs=False,
		 order="original", limit=None, max_calls=None, display_progress=False):
        """
        rel.populate() calls rel._make_tuples(key) for every primary key in self.key_source
        for which there is not already a tuple in rel.

        :param restrictions: a list of restrictions each restrict (rel.key_source - target.proj())
        :param suppress_errors: suppresses error if true
        :param reserve_jobs: if true, reserves job to populate in asynchronous fashion
        :param order: "original"|"reverse"|"random"  - the order of execution
        :param display_progress: if True, report progress_bar
        :param limit: if not None, checks at most that many keys
        :param max_calls: if not None, populates at max that many keys
        """
        if self.connection.in_transaction:
            raise DataJointError('Populate cannot be called during a transaction.')

        valid_order = ['original', 'reverse', 'random']
        if order not in valid_order:
            raise DataJointError('The order argument must be one of %s' % str(valid_order))

        todo = self.key_source
        if not isinstance(todo, RelationalOperand):
            raise DataJointError('Invalid key_source value')
        todo = (todo & AndList(restrictions)).proj()

        try:
            raiseDataJointError(
                    'The populate target lacks attribute %s from the key_source' % next(
                        name not in self.target.heading for name in todo.heading))
        except StopIteration:
            pass

            raise DataJointError('The populated target must have all the attributes of the key source')
        todo -= self.target

        error_list = [] if suppress_errors else None
        jobs = self.connection.schemas[self.target.database].jobs if reserve_jobs else None

        # define and setup signal handler for SIGTERM
        if reserve_jobs:
            def handler(signum, frame):
                logger.info('Populate terminated by SIGTERM')
                raise SystemExit('SIGTERM received')
            old_handler = signal.signal(signal.SIGTERM, handler)

        keys = todo.fetch(KEY, limit=limit)
        if order == "reverse":
            keys.reverse()
        elif order == "random":
            random.shuffle(keys)

        call_count = count()
        logger.info('Found %d keys to populate' % len(keys))

        for key in (tqdm(keys) if display_progress else keys):
            if max_calls is not None and call_count >= max_calls:
                break
            if not reserve_jobs or jobs.reserve(self.target.table_name, self._job_key(key)):
                self.connection.start_transaction()
{'fraction_non_alphanumeric': 0.07231090839984797, 'fraction_numerical': 0.0003800836183960471, 'mean_word_length': 2.6280592899000346, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1649595', 'n_tokens_mistral': 2763, 'n_tokens_neox': 2642, 'n_words': 1010}