<commit_before>"""SQLite DB engine / backend"""

# Copyright (c) 2018 <NAME>.  This is free software released
# under the MIT License.  See `LICENSE.txt` for details.


import copy
import sqlite3
import sys

from barnapy import logging

from .. import database
from .. import general
from .. import records as recs


def placeholders_for_params(n_params):
    return '(?' + (', ?' * (n_params - 1)) + ')'

def gen_fetchmany(cursor):
    rows = cursor.fetchmany()
    while rows:
        yield from rows
        rows = cursor.fetchmany()

python2sqlite_types = {
    bytes: 'blob',
    float: 'real',
    int: 'integer',
    # SQLite allows a column definition without a specific type
    object: '',
    str: 'text',
    }


class SqliteDb(database.Database):
    """Represents a SQLite database and provides schema-level operations"""

    def __init__( # TODO max mem, max threads, other options
            self,
            filename=None,
            sqlite_array_size=(8 * 2 ** 10), # 8Ki
    ):
        self._filename = (filename
                          if filename is not None
                          else ':memory:')
        self._sqlite_array_size = sqlite_array_size
        repr_id = repr(self) + '@' + hex(id(self))
        self._logger = logging.getLogger(repr_id)
        self._logger.info(
            'Opening connection to Sqlite DB: {}'
            .format(self._filename))
        # TODO open and maintain a connection at the object level but what about committing, closing the connection, etc.?
        self._connection = sqlite3.connect(self._filename)
        self._logger.info('Connected')
        self._tables = {} # References to tables are circular

    def close(self):
        """Close and invalidate the database connection"""
        # Bail if already closed or not set up
        if self._connection is None:
            return
        self._logger.info('Closing DB connection')
        # Clear circular references to tables
        if self._tables:
            for table in self._tables.values():
                table.disconnect()
            self._tables.clear()
        # Close the DB connection
        try:
            self._connection.close()
        except Exception as e:
            self._logger.exception(
                'Failed to close DB connection: {}'.format(e))
        else:
            self._logger.info('DB connection closed')
        finally:
            # Release reference no matter what
            self._connection = None

    __del__ = close

    @property
    def name(self):
        return self._filename

    def __repr__(self):
        return '{}({!r})'.format(
            general.fq_typename(self), self._filename)

    def execute_query(self, query, parameters=None):
        self._logger.info(
            'Executing query: {}; parameters: {}'
            .format(query, parameters))
        cursor = self._connection.cursor()
        cursor.arraysize = self._sqlite_array_size
        if parameters is None:
            cursor.execute(query)
        else:
            cursor.execute(query, parameters)
        return cursor

    def execute_many(self, query, parameters):
        self._logger.info(
            'Executing query: {}; parameters: {}'
            .format(query, parameters))
        cursor = self._connection.cursor()
        cursor.arraysize = self._sqlite_array_size
        cursor.executemany(query, parameters)
        return cursor

    def _catalog_table_name(self, namespace):
        # Construct and check the catalog table name
        catalog_table_name = namespace + '.sqlite_master'
        return self.interpret_sql_name(catalog_table_name)

    def interpret_sql_name(self, name):
        dotted_name = super().interpret_sql_name(name)
        # Check that the dotted name has no more than 2 components
        if len(dotted_name) > 2:
            raise database.DbSyntaxError(
                'Too many name components: {}'.format(name))
        return dotted_name

    def _process_name(self, name):
        # Parse and check name
        dotted_name = self.interpret_sql_name(name)
        # Split à la dirname, basename
        namespace, obj_name = dotted_name.rest_last()
        # Default to `main` namespace
        if namespace is None:
            namespace = 'main'
        # Return it all
        return dotted_name, namespace, obj_name

    def namespaces(self):
        yield from (db_name
                    for db_id, db_name, filename
                    in gen_fetchmany(
                        self.execute_query('pragma database_list')))

    _list_objects_sql = 'select name, type, sql from {}'
    _list_objects_with_types_sql = (
        'select name, type, sql from {} where type in {}')

    def objects(self, types=None, namespace=None):
        """Return an iterable of DB objects as (name, type, sql) tuples"""
        # Default to `main` namespace
        if namespace is None:
            namespace = 'main'
        catalog_table_name = self._catalog_table_name(namespace)
        # Handle no type, a single type, or an iterable of types
        if types is None:
            parameters = None
            query = self._list_objects_sql.format(
                database.quote_name(catalog_table_name.name))
        elif hasattr(types, '__iter__') and not isinstance(types, str):
            parameters = tuple(
                database.DbObjectType.convert(o).name for o in types)
            query = self._list_objects_with_types_sql.format(
                database.quote_name(catalog_table_name.name),
                placeholders_for_params(len(parameters)))
        else:
            parameters = (database.DbObjectType.convert(types).name,)
            query = self._list_objects_with_types_sql.format(
                database.quote_name(catalog_table_name.name),
                placeholders_for_params(len(parameters)))
        # Generate the objects as (name, type, sql) tuples
        yield from gen_fetchmany(self.execute_query(query, parameters))

    _schema_sql = 'select sql from {} where name = ?'

    def schema(self, name):
        dotted_name, namespace, obj_name = self._process_name(name)
        catalog_table_name = self._catalog_table_name(namespace)
        query = self._schema_sql.format(
            database.quote_name(catalog_table_name.name))
        schemas = list(<commit_msg>Removed unnecessary generator that wraps another generator.<commit_after>"""SQLite DB engine / backend"""

# Copyright (c) 2018 <NAME>.  This is free software released
# under the MIT License.  See `LICENSE.txt` for details.


import copy
import sqlite3
import sys

from barnapy import logging

from .. import database
from .. import general
from .. import records as recs


def placeholders_for_params(n_params):
    return '(?' + (', ?' * (n_params - 1)) + ')'

def gen_fetchmany(cursor):
    rows = cursor.fetchmany()
    while rows:
        yield from rows
        rows = cursor.fetchmany()

python2sqlite_types = {
    bytes: 'blob',
    float: 'real',
    int: 'integer',
    # SQLite allows a column definition without a specific type
    object: '',
    str: 'text',
    }


class SqliteDb(database.Database):
    """Represents a SQLite database and provides schema-level operations"""

    def __init__( # TODO max mem, max threads, other options
            self,
            filename=None,
            sqlite_array_size=(8 * 2 ** 10), # 8Ki
    ):
        self._filename = (filename
                          if filename is not None
                          else ':memory:')
        self._sqlite_array_size = sqlite_array_size
        repr_id = repr(self) + '@' + hex(id(self))
        self._logger = logging.getLogger(repr_id)
        self._logger.info(
            'Opening connection to Sqlite DB: {}'
            .format(self._filename))
        # TODO open and maintain a connection at the object level but what about committing, closing the connection, etc.?
        self._connection = sqlite3.connect(self._filename)
        self._logger.info('Connected')
        self._tables = {} # References to tables are circular

    def close(self):
        """Close and invalidate the database connection"""
        # Bail if already closed or not set up
        if self._connection is None:
            return
        self._logger.info('Closing DB connection')
        # Clear circular references to tables
        if self._tables:
            for table in self._tables.values():
                table.disconnect()
            self._tables.clear()
        # Close the DB connection
        try:
            self._connection.close()
        except Exception as e:
            self._logger.exception(
                'Failed to close DB connection: {}'.format(e))
        else:
            self._logger.info('DB connection closed')
        finally:
            # Release reference no matter what
            self._connection = None

    __del__ = close

    @property
    def name(self):
        return self._filename

    def __repr__(self):
        return '{}({!r})'.format(
            general.fq_typename(self), self._filename)

    def execute_query(self, query, parameters=None):
        self._logger.info(
            'Executing query: {}; parameters: {}'
            .format(query, parameters))
        cursor = self._connection.cursor()
        cursor.arraysize = self._sqlite_array_size
        if parameters is None:
            cursor.execute(query)
        else:
            cursor.execute(query, parameters)
        return cursor

    def execute_many(self, query, parameters):
        self._logger.info(
            'Executing query: {}; parameters: {}'
            .format(query, parameters))
        cursor = self._connection.cursor()
        cursor.arraysize = self._sqlite_array_size
        cursor.executemany(query, parameters)
        return cursor

    def _catalog_table_name(self, namespace):
        # Construct and check the catalog table name
        catalog_table_name = namespace + '.sqlite_master'
        return self.interpret_sql_name(catalog_table_name)

    def interpret_sql_name(self, name):
        dotted_name = super().interpret_sql_name(name)
        # Check that the dotted name has no more than 2 components
        if len(dotted_name) > 2:
            raise database.DbSyntaxError(
                'Too many name components: {}'.format(name))
        return dotted_name

    def _process_name(self, name):
        # Parse and check name
        dotted_name = self.interpret_sql_name(name)
        # Split à la dirname, basename
        namespace, obj_name = dotted_name.rest_last()
        # Default to `main` namespace
        if namespace is None:
            namespace = 'main'
        # Return it all
        return dotted_name, namespace, obj_name

    def namespaces(self):
        yield from (db_name
                    for db_id, db_name, filename
                    in gen_fetchmany(
                        self.execute_query('pragma database_list')))

    _list_objects_sql = 'select name, type, sql from {}'
    _list_objects_with_types_sql = (
        'select name, type, sql from {} where type in {}')

    def objects(self, types=None, namespace=None):
        """Return an iterable of DB objects as (name, type, sql) tuples"""
        # Default to `main` namespace
        if namespace is None:
            namespace = 'main'
        catalog_table_name = self._catalog_table_name(namespace)
        # Handle no type, a single type, or an iterable of types
        if types is None:
            parameters = None
            query = self._list_objects_sql.format(
                database.quote_name(catalog_table_name.name))
        elif hasattr(types, '__iter__') and not isinstance(types, str):
            parameters = tuple(
                database.DbObjectType.convert(o).name for o in types)
            query = self._list_objects_with_types_sql.format(
                database.quote_name(catalog_table_name.name),
                placeholders_for_params(len(parameters)))
        else:
            parameters = (database.DbObjectType.convert(types).name,)
            query = self._list_objects_with_types_sql.format(
                database.quote_name(catalog_table_name.name),
                placeholders_for_params(len(parameters)))
        # Generate the objects as (name, type, sql) tuples
        return gen_fetchmany(self.execute_query(query, parameters))

    _schema_sql = 'select sql from {} where name = ?'

    def schema(self, name):
        dotted_name, namespace, obj_name = self._process_name(name)
        catalog_table_name = self._catalog_table_name(namespace)
        query = self._schema_sql.format(
            database.quote_name(catalog_table_name.name))
        schemas = list(
{'fraction_non_alphanumeric': 0.0906019173345906, 'fraction_numerical': 0.0023573785950023575, 'mean_word_length': 2.2293834052270998, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4008225', 'n_tokens_mistral': 3456, 'n_tokens_neox': 3331, 'n_words': 968}