<commit_before>import collections
import itertools
import inspect
import platform
import numpy as np
import pymysql
import logging
from . import config, DataJointError
from .declare import declare
from .relational_operand import RelationalOperand
from .blob import pack
from .utils import user_choice
from .heading import Heading
from .settings import server_error_codes
from . import __version__ as version

logger = logging.getLogger(__name__)


class BaseRelation(RelationalOperand):
    """
    BaseRelation is an abstract class that represents a base relation, i.e. a table in the database.
    To make it a concrete class, override the abstract properties specifying the connection,
    table name, database, context, and definition.
    A Relation implements insert and delete methods in addition to inherited relational operators.
    """
    _heading = None
    _context = None
    database = None
    _log_ = None

    # -------------- required by RelationalOperand ----------------- #
    @property
    def heading(self):
        """
        Returns the table heading. If the table is not declared, attempts to declare it and return heading.
        :return: table heading
        """
        if self._heading is None:
            self._heading = Heading()  # instance-level heading
        if not self._heading:  # lazy loading of heading
            self._heading.init_from_database(self.connection, self.database, self.table_name)
        return self._heading

    @property
    def context(self):
        return self._context

    def declare(self):
        """
        Loads the table heading. If the table is not declared, use self.definition to declare
        """
        try:
            self.connection.query(
                declare(self.full_table_name, self.definition, self._context))
        except pymysql.OperationalError as error:
            if error.args[0] == server_error_codes['command denied']:
                logger.warning(error.args[1])
        else:
            self._log('Declared ' + self.full_table_name)

    @property
    def from_clause(self):
        """
        :return: the FROM clause of SQL SELECT statements.
        """
        return self.full_table_name

    def get_select_fields(self, select_fields=None):
        """
        :return: the selected attributes from the SQL SELECT statement.
        """
        return '*' if select_fields is None else self.heading.project(select_fields).as_sql

    def parents(self, primary=None):
        """
        :param primary: if None, then all parents are returned. If True, then only foreign keys composed of
            primary key attributes are considered.  If False, the only foreign keys including at least one non-primary
            attribute are considered.
        :return: dict of tables referenced with self's foreign keys
        """
        return self.connection.dependencies.parents(self.full_table_name, primary)

    def children(self, primary=None):
        """
        :param primary: if None, then all parents are returned. If True, then only foreign keys composed of
            primary key attributes are considered.  If False, the only foreign keys including at least one non-primary
            attribute are considered.
        :return: dict of tables with foreign keys referencing self
        """
        return self.connection.dependencies.children(self.full_table_name, primary)

    @property
    def is_declared(self):
        """
        :return: True is the table is declared in the database
        """
        return self.connection.query(
            'SHOW TABLES in `{database}` LIKE "{table_name}"'.format(
                database=self.database, table_name=self.table_name)).rowcount > 0

    @property
    def full_table_name(self):
        """
        :return: full table name in the database
        """
        return r"`{0:s}`.`{1:s}`".format(self.database, self.table_name)

    @property
    def _log(self):
        if self._log_ is None:
            self._log_ = Log(self.connection, database=self.database)
        return self._log_

    def insert1(self, row, **kwargs):
        """
        Insert one data record or one Mapping (like a dict).
        :param row: a numpy record, a dict-like object, or an ordered sequence to be inserted as one row.
        For kwargs, see insert()
        """
        self.insert((row,), **kwargs)

    def insert(self, rows, replace=False, ignore_errors=False, skip_duplicates=False, ignore_extra_fields=False):
        """
        Insert a collection of rows.

        :param rows: An iterable where an element is a numpy record, a dict-like object, or an ordered sequence.
            rows may also be another relation with the same heading.
        :param replace: If True, replaces the existing tuple.
        :param ignore_errors: If True, ignore errors: e.g. constraint violations.
        :param skip_duplicates: If True, silently skip duplicate inserts.
        :param ignore_extra_fields: If False, fields that are not in the heading raise error.

        Example::
        >>> relation.insert([
        >>>     dict(subject_id=7, species="mouse", date_of_birth="2014-09-01"),
        >>>     dict(subject_id=8, species="mouse", date_of_birth="2014-09-02")])
        """

        heading = self.heading
        if isinstance(rows, RelationalOperand):
            # insert from select
            if not ignore_extra_fields:
		try:
		    raise DataJointError("Attribute %s not found.", 
                        next(name for name in rows.heading if name not in heading))
                except StopIteration:
                    pass
            fields='`'+'`,`'.join(name for name in heading if name in rows.heading) + '`'
	    query = 'INSERT{ignore} INTO {table} ({fields}) {select}'.format( 
	       ignore=" IGNORE" if ignore_errors or skip_duplicates else "",
	       fields=fields,
	       table=self.full_table_name,
	       select=rows.make_sql(select_fields=fields)))
            self.connection.query(query)
            return 

        if heading.attributes is None:
            logger.warning('Could not access table {table}'.format(table=self.full_table_name))
            return

        field_list = None  # ensures that all rows have the same attributes in the same order as the first row.

        def make_row_to_insert(row):
            """
            :param row:  A tuple to insert
            :return: a dict with fields 'names', 'placeholders', 'values'
            """

            def make_placeholder(name, value):
                """
                For a given attribute `name` with `value`, return its processed value or value placeholder
                as a string to be included in the query and the value, if any, to be submitted for
                processing by mysql API.
                :param name:
                :param value:
                """
                if ignore_extra_fields and name not in heading:
                    return None
                if heading[name].is_blob:
                    value = pack(value)
                    placeholder = '%s'
                elif heading[name].numeric:
                    if value is None or value == '' or np.isnan(np.float(value)):  # nans are turned into NULLs
                        placeholder = 'NULL'<commit_msg>minor formatting<commit_after>import collections
import itertools
import inspect
import platform
import numpy as np
import pymysql
import logging
from . import config, DataJointError
from .declare import declare
from .relational_operand import RelationalOperand
from .blob import pack
from .utils import user_choice
from .heading import Heading
from .settings import server_error_codes
from . import __version__ as version

logger = logging.getLogger(__name__)


class BaseRelation(RelationalOperand):
    """
    BaseRelation is an abstract class that represents a base relation, i.e. a table in the database.
    To make it a concrete class, override the abstract properties specifying the connection,
    table name, database, context, and definition.
    A Relation implements insert and delete methods in addition to inherited relational operators.
    """
    _heading = None
    _context = None
    database = None
    _log_ = None

    # -------------- required by RelationalOperand ----------------- #
    @property
    def heading(self):
        """
        Returns the table heading. If the table is not declared, attempts to declare it and return heading.
        :return: table heading
        """
        if self._heading is None:
            self._heading = Heading()  # instance-level heading
        if not self._heading:  # lazy loading of heading
            self._heading.init_from_database(self.connection, self.database, self.table_name)
        return self._heading

    @property
    def context(self):
        return self._context

    def declare(self):
        """
        Loads the table heading. If the table is not declared, use self.definition to declare
        """
        try:
            self.connection.query(
                declare(self.full_table_name, self.definition, self._context))
        except pymysql.OperationalError as error:
            if error.args[0] == server_error_codes['command denied']:
                logger.warning(error.args[1])
        else:
            self._log('Declared ' + self.full_table_name)

    @property
    def from_clause(self):
        """
        :return: the FROM clause of SQL SELECT statements.
        """
        return self.full_table_name

    def get_select_fields(self, select_fields=None):
        """
        :return: the selected attributes from the SQL SELECT statement.
        """
        return '*' if select_fields is None else self.heading.project(select_fields).as_sql

    def parents(self, primary=None):
        """
        :param primary: if None, then all parents are returned. If True, then only foreign keys composed of
            primary key attributes are considered.  If False, the only foreign keys including at least one non-primary
            attribute are considered.
        :return: dict of tables referenced with self's foreign keys
        """
        return self.connection.dependencies.parents(self.full_table_name, primary)

    def children(self, primary=None):
        """
        :param primary: if None, then all parents are returned. If True, then only foreign keys composed of
            primary key attributes are considered.  If False, the only foreign keys including at least one non-primary
            attribute are considered.
        :return: dict of tables with foreign keys referencing self
        """
        return self.connection.dependencies.children(self.full_table_name, primary)

    @property
    def is_declared(self):
        """
        :return: True is the table is declared in the database
        """
        return self.connection.query(
            'SHOW TABLES in `{database}` LIKE "{table_name}"'.format(
                database=self.database, table_name=self.table_name)).rowcount > 0

    @property
    def full_table_name(self):
        """
        :return: full table name in the database
        """
        return r"`{0:s}`.`{1:s}`".format(self.database, self.table_name)

    @property
    def _log(self):
        if self._log_ is None:
            self._log_ = Log(self.connection, database=self.database)
        return self._log_

    def insert1(self, row, **kwargs):
        """
        Insert one data record or one Mapping (like a dict).
        :param row: a numpy record, a dict-like object, or an ordered sequence to be inserted as one row.
        For kwargs, see insert()
        """
        self.insert((row,), **kwargs)

    def insert(self, rows, replace=False, ignore_errors=False, skip_duplicates=False, ignore_extra_fields=False):
        """
        Insert a collection of rows.

        :param rows: An iterable where an element is a numpy record, a dict-like object, or an ordered sequence.
            rows may also be another relation with the same heading.
        :param replace: If True, replaces the existing tuple.
        :param ignore_errors: If True, ignore errors: e.g. constraint violations.
        :param skip_duplicates: If True, silently skip duplicate inserts.
        :param ignore_extra_fields: If False, fields that are not in the heading raise error.

        Example::
        >>> relation.insert([
        >>>     dict(subject_id=7, species="mouse", date_of_birth="2014-09-01"),
        >>>     dict(subject_id=8, species="mouse", date_of_birth="2014-09-02")])
        """

        heading = self.heading
        if isinstance(rows, RelationalOperand):
            # insert from select
            if not ignore_extra_fields:
                try:
                    raise DataJointError("Attribute %s not found.", 
                        next(name for name in rows.heading if name not in heading))
                except StopIteration:
                    pass
            fields='`'+'`,`'.join(name for name in heading if name in rows.heading) + '`'
            query = 'INSERT{ignore} INTO {table} ({fields}) {select}'.format( 
               ignore=" IGNORE" if ignore_errors or skip_duplicates else "",
               fields=fields,
               table=self.full_table_name,
               select=rows.make_sql(select_fields=fields)))
            self.connection.query(query)
            return 

        if heading.attributes is None:
            logger.warning('Could not access table {table}'.format(table=self.full_table_name))
            return

        field_list = None  # ensures that all rows have the same attributes in the same order as the first row.

        def make_row_to_insert(row):
            """
            :param row:  A tuple to insert
            :return: a dict with fields 'names', 'placeholders', 'values'
            """

            def make_placeholder(name, value):
                """
                For a given attribute `name` with `value`, return its processed value or value placeholder
                as a string to be included in the query and the value, if any, to be submitted for
                processing by mysql API.
                :param name:
                :param value:
                """
                if ignore_extra_fields and name not in heading:
                    return None
                if heading[name].is_blob:
                    value = pack(value)
                    placeholder = '%s'
                elif heading[name].numeric:
                    if value is None or value == '' or np.isnan(np.float(value)):  # nans are turned into NULLs
                        placeholder = 'NULL'
{'fraction_non_alphanumeric': 0.08565981206591311, 'fraction_numerical': 0.0032684189023559854, 'mean_word_length': 2.194909723732869, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6480676', 'n_tokens_mistral': 3774, 'n_tokens_neox': 3623, 'n_words': 1352}