<commit_before>import logging as logger

from django.apps import apps
from kolibri.content.apps import KolibriContentConfig
from kolibri.content.legacy_models import License
from kolibri.content.models import CONTENT_SCHEMA_VERSION, NO_VERSION, ChannelMetadata, ContentNode, ContentTag, File, Language, LocalFile
from kolibri.utils.time import local_now
from sqlalchemy.exc import SQLAlchemyError

from .annotation import set_leaf_node_availability_from_local_file_availability
from .channels import read_channel_metadata_from_db_file
from .paths import get_content_database_file_path
from .sqlalchemybridge import Bridge, ClassNotFoundError

logging = logger.getLogger(__name__)

CONTENT_APP_NAME = KolibriContentConfig.label

merge_models = [
    ContentTag,
    LocalFile,
    Language,
]

class ChannelImport(object):
    """
    The ChannelImport class has two functions:

    1) it acts as the default import pattern for importing content databases that have naively compatible version
    with the current version of Kolibri (i.e. no explicit mappings are required to bring data from the content db
    into the main db, as there is a one to one correspondence in table names and column names within tables).

    2) It is also the base class for any more complex import that requires explicit schema mappings from one version
    to another.
    """

    # Specific instructions and exceptions for importing table from previous versions of Kolibri
    # Mappings can be:
    # 1) 'per_row', specifying mappings for an entire row, string can either be an attribute
    #    or a method name on the import class
    # 2) 'per_table' mapping an entire table at a time. Only a method name can be used for 'per_table' mappings.
    #
    # Both can be used simultaneously.
    #
    # See NoVersionChannelImport for an annotated example.
    schema_mapping = {}

    def __init__(self, channel_id):
        self.channel_id = channel_id

        self.source = Bridge(sqlite_file_path=get_content_database_file_path(channel_id))

        self.destination = Bridge(app_name=CONTENT_APP_NAME)

        content_app = apps.get_app_config(CONTENT_APP_NAME)

        # Use this rather than get_models, as it returns a list of all models, including those
        # generated by ManyToMany fields, whereas get_models only returns explicitly defined
        # Model classes
        self.content_models = list(content_app.get_models(include_auto_created=True))

        # Get the next available tree_id in our database
        self.tree_id = self.find_unique_tree_id()

    def get_all_destination_tree_ids(self):
        ContentNodeRecord = self.destination.get_class(ContentNode)
        return sorted(map(lambda x: x[0], self.destination.session.query(
            ContentNodeRecord.tree_id).distinct().all()))

    def find_unique_tree_id(self):
        tree_ids = self.get_all_destination_tree_ids()
        # If there are no pre-existing tree_ids just escape here and return 1
        if not tree_ids:
            return 1
        if len(tree_ids) == 1:
            if tree_ids[0] == 1:
                return 2
            return 1

        # Do a binary search to find the lowest unused tree_id
        def find_hole_in_list(ids):
            last = len(ids) - 1
            middle = int(last/2 + 1)
            # Check if the lower half of ids has a hole in it
            if ids[middle] - ids[0] != middle:
                # List is only two ids, so hole must be between them
                if middle == 1:
                    return ids[0] + 1
                return find_hole_in_list(ids[:middle])
            # Otherwise check if there is a hole in the second half
            if ids[last] - ids[middle] != last - middle:
                # Second half is only two ids so hole must be between them
                if last - middle == 1:
                    return ids[middle] + 1
                return find_hole_in_list(ids[middle:])
            # We should only reach this point in the first iteration, if there are no holes in either
            # the first or the last half of the list, therefore, we just take the max of the list plus 1
            # Because the list is already sorted, we can just take the last value
            return ids[-1] + 1
        return find_hole_in_list(tree_ids)

    def generate_row_mapper(self, mappings=None):
        # If no mappings, just use an empty object
        if mappings is None:
            mappings = {}

        def mapper(record, column):
            """
            A mapper function for the mappings object
            """
            if column in mappings:
                # If the column name is in our defined mappings object,
                # then we need to try to find an alternate value
                col_map = mappings.get(column)  # Get the string value for the mapping
                if hasattr(record, col_map):
                    # Is this mapping value another column of the table?
                    # If so, return it straight away
                    return getattr(record, col_map)
                elif hasattr(self, col_map):
                    # Otherwise, check to see if the import class has an attribute with this name
                    # We assume that if it is, then it is a callable method that accepts the row
                    # data as its only argument, if so, return the result of calling that method
                    # on the row data
                    return getattr(self, col_map)(record)
                else:
                    # If neither of these true, we specified a column mapping that is invalid
                    raise AttributeError('Column mapping specified but no valid column name or method found')
            else:
                # Otherwise, we can just get the value directly from the record
                return getattr(record, column, None)
        # Return the mapper function for repeated use
        return mapper

    def base_table_mapper(self, SourceRecord):
        # If SourceRecord is none, then the source table does not exist in the DB
        if SourceRecord:
            return self.source.session.query(SourceRecord).all()
        return []

    def generate_table_mapper(self, table_map=None):
        if table_map is None:
            # If no table mapping specified, just use the default
            return self.base_table_mapper
        # Can only be a method on the Import object
        if hasattr(self, table_map):
            # If it is a method of the import class return that method for later use
            return getattr(self, table_map)
        # If we got here, there is an invalid table mapping
        raise AttributeError('Table mapping specified but no valid method found')

    def table_import(self, model, row_mapper, table_mapper, unflushed_rows):
        DestinationRecord = self.destination.get_class(model)
        dest_table = DestinationRecord.__table__

        # If the source class does not exist (i.e. this table is undefined in the source database)
        # this will raise an error so we set it to None. In this case, a custom table mapper must
        # have been set up to handle the fact that this is None.
        try:
            SourceRecord = self.source.get_class(model)
        except ClassNotFoundError:
            SourceRecord = None

        columns = dest_table.columns.keys()
        data_to_insert = []
        merge = model in merge_models
        for record in table_mapper(SourceRecord):
            data = {
                str(column): row_mapper(record, column) for column in columns if row_mapper(record, column) is not None
            }
            if merge:
                # Models that should be merged (see list above) need to be individually merged into the session
                # as SQL Alchemy ORM does not support INSERT ... ON DUPLICATE KEY UPDATE style queries,
                # as not available in SQLite, only MySQL as far as I can tell:
                # http://hackthology.com/how-to-compile-mysqls-on-duplicate-key-update-in-sql-alchemy.html
                self.destination.session.merge(DestinationRecord(**data))
            else:
                data_to_insert.append(data)
            unflushed_rows += 1
            if unflushed_rows == 10000:
                if not merge:
                    self.destination.session.bulk_insert_mappings(DestinationRecord, data_to_insert)
                    data_to_insert = []
                self.destination.session.flush()
                unflushed_rows = 0
        if not merge and data_to_insert:
            self.destination.session.bulk_insert_mappings(DestinationRecord, data_to_insert)
        return unflushed_rows

    def import_channel_data(self):

        unflushed_rows = 0

        try:
            for model in self.content_models:
                mapping = self.schema_mapping.get(model, {})
                row_mapper = self.generate_row_mapper(mapping.get('per_row'))
                table_mapper = self.generate_table_mapper(mapping.get('per_table'))
                logging.info('Importing {model} data'.format(model=model.__name__))
                unflushed_rows = self.table_import(model, row_mapper, table_mapper, unflushed_rows)
            self.destination.session.commit()

        except SQLAlchemyError:
            # Rollback the transaction if any error occurs during the transaction
            self.destination.session.rollback()

    def end(self):
        self.source.end()
        self.destination.end()


class NoVersionChannelImport(ChannelImport):
    """
    Class defining the schema mapping for importing old content databases (i.e. ones produced before the
    ChannelImport machinery was implemented). The schema mapping below defines how to bring in information
    from the old version of the Kolibri content databases into the database for the current version of Kolibri.
    """

    schema_mapping = {
        # The top level keys of the schema_mapping are the Content Django Models that are to be imported
        ContentNode: {
            # For each model's mappings, can defined both 'per_row' and 'per_table' mappings.
            'per_row': {
                # The key of the 'per_row' mapping object is the table column that we are populating
                # In the case of Django ForeignKey fields, this will be the field name plus _id
                # The value is a string that refers either to a table column on the source data
                # or a method on this import class that will be passed the row data and should return<commit_msg>Reraise exceptions to prevent code from running assuming that the operation succeeded.<commit_after>import logging as logger

from django.apps import apps
from kolibri.content.apps import KolibriContentConfig
from kolibri.content.legacy_models import License
from kolibri.content.models import CONTENT_SCHEMA_VERSION, NO_VERSION, ChannelMetadata, ContentNode, ContentTag, File, Language, LocalFile
from kolibri.utils.time import local_now
from sqlalchemy.exc import SQLAlchemyError

from .annotation import set_leaf_node_availability_from_local_file_availability
from .channels import read_channel_metadata_from_db_file
from .paths import get_content_database_file_path
from .sqlalchemybridge import Bridge, ClassNotFoundError

logging = logger.getLogger(__name__)

CONTENT_APP_NAME = KolibriContentConfig.label

merge_models = [
    ContentTag,
    LocalFile,
    Language,
]

class ChannelImport(object):
    """
    The ChannelImport class has two functions:

    1) it acts as the default import pattern for importing content databases that have naively compatible version
    with the current version of Kolibri (i.e. no explicit mappings are required to bring data from the content db
    into the main db, as there is a one to one correspondence in table names and column names within tables).

    2) It is also the base class for any more complex import that requires explicit schema mappings from one version
    to another.
    """

    # Specific instructions and exceptions for importing table from previous versions of Kolibri
    # Mappings can be:
    # 1) 'per_row', specifying mappings for an entire row, string can either be an attribute
    #    or a method name on the import class
    # 2) 'per_table' mapping an entire table at a time. Only a method name can be used for 'per_table' mappings.
    #
    # Both can be used simultaneously.
    #
    # See NoVersionChannelImport for an annotated example.
    schema_mapping = {}

    def __init__(self, channel_id):
        self.channel_id = channel_id

        self.source = Bridge(sqlite_file_path=get_content_database_file_path(channel_id))

        self.destination = Bridge(app_name=CONTENT_APP_NAME)

        content_app = apps.get_app_config(CONTENT_APP_NAME)

        # Use this rather than get_models, as it returns a list of all models, including those
        # generated by ManyToMany fields, whereas get_models only returns explicitly defined
        # Model classes
        self.content_models = list(content_app.get_models(include_auto_created=True))

        # Get the next available tree_id in our database
        self.tree_id = self.find_unique_tree_id()

    def get_all_destination_tree_ids(self):
        ContentNodeRecord = self.destination.get_class(ContentNode)
        return sorted(map(lambda x: x[0], self.destination.session.query(
            ContentNodeRecord.tree_id).distinct().all()))

    def find_unique_tree_id(self):
        tree_ids = self.get_all_destination_tree_ids()
        # If there are no pre-existing tree_ids just escape here and return 1
        if not tree_ids:
            return 1
        if len(tree_ids) == 1:
            if tree_ids[0] == 1:
                return 2
            return 1

        # Do a binary search to find the lowest unused tree_id
        def find_hole_in_list(ids):
            last = len(ids) - 1
            middle = int(last/2 + 1)
            # Check if the lower half of ids has a hole in it
            if ids[middle] - ids[0] != middle:
                # List is only two ids, so hole must be between them
                if middle == 1:
                    return ids[0] + 1
                return find_hole_in_list(ids[:middle])
            # Otherwise check if there is a hole in the second half
            if ids[last] - ids[middle] != last - middle:
                # Second half is only two ids so hole must be between them
                if last - middle == 1:
                    return ids[middle] + 1
                return find_hole_in_list(ids[middle:])
            # We should only reach this point in the first iteration, if there are no holes in either
            # the first or the last half of the list, therefore, we just take the max of the list plus 1
            # Because the list is already sorted, we can just take the last value
            return ids[-1] + 1
        return find_hole_in_list(tree_ids)

    def generate_row_mapper(self, mappings=None):
        # If no mappings, just use an empty object
        if mappings is None:
            mappings = {}

        def mapper(record, column):
            """
            A mapper function for the mappings object
            """
            if column in mappings:
                # If the column name is in our defined mappings object,
                # then we need to try to find an alternate value
                col_map = mappings.get(column)  # Get the string value for the mapping
                if hasattr(record, col_map):
                    # Is this mapping value another column of the table?
                    # If so, return it straight away
                    return getattr(record, col_map)
                elif hasattr(self, col_map):
                    # Otherwise, check to see if the import class has an attribute with this name
                    # We assume that if it is, then it is a callable method that accepts the row
                    # data as its only argument, if so, return the result of calling that method
                    # on the row data
                    return getattr(self, col_map)(record)
                else:
                    # If neither of these true, we specified a column mapping that is invalid
                    raise AttributeError('Column mapping specified but no valid column name or method found')
            else:
                # Otherwise, we can just get the value directly from the record
                return getattr(record, column, None)
        # Return the mapper function for repeated use
        return mapper

    def base_table_mapper(self, SourceRecord):
        # If SourceRecord is none, then the source table does not exist in the DB
        if SourceRecord:
            return self.source.session.query(SourceRecord).all()
        return []

    def generate_table_mapper(self, table_map=None):
        if table_map is None:
            # If no table mapping specified, just use the default
            return self.base_table_mapper
        # Can only be a method on the Import object
        if hasattr(self, table_map):
            # If it is a method of the import class return that method for later use
            return getattr(self, table_map)
        # If we got here, there is an invalid table mapping
        raise AttributeError('Table mapping specified but no valid method found')

    def table_import(self, model, row_mapper, table_mapper, unflushed_rows):
        DestinationRecord = self.destination.get_class(model)
        dest_table = DestinationRecord.__table__

        # If the source class does not exist (i.e. this table is undefined in the source database)
        # this will raise an error so we set it to None. In this case, a custom table mapper must
        # have been set up to handle the fact that this is None.
        try:
            SourceRecord = self.source.get_class(model)
        except ClassNotFoundError:
            SourceRecord = None

        columns = dest_table.columns.keys()
        data_to_insert = []
        merge = model in merge_models
        for record in table_mapper(SourceRecord):
            data = {
                str(column): row_mapper(record, column) for column in columns if row_mapper(record, column) is not None
            }
            if merge:
                # Models that should be merged (see list above) need to be individually merged into the session
                # as SQL Alchemy ORM does not support INSERT ... ON DUPLICATE KEY UPDATE style queries,
                # as not available in SQLite, only MySQL as far as I can tell:
                # http://hackthology.com/how-to-compile-mysqls-on-duplicate-key-update-in-sql-alchemy.html
                self.destination.session.merge(DestinationRecord(**data))
            else:
                data_to_insert.append(data)
            unflushed_rows += 1
            if unflushed_rows == 10000:
                if not merge:
                    self.destination.session.bulk_insert_mappings(DestinationRecord, data_to_insert)
                    data_to_insert = []
                self.destination.session.flush()
                unflushed_rows = 0
        if not merge and data_to_insert:
            self.destination.session.bulk_insert_mappings(DestinationRecord, data_to_insert)
        return unflushed_rows

    def import_channel_data(self):

        unflushed_rows = 0

        try:
            for model in self.content_models:
                mapping = self.schema_mapping.get(model, {})
                row_mapper = self.generate_row_mapper(mapping.get('per_row'))
                table_mapper = self.generate_table_mapper(mapping.get('per_table'))
                logging.info('Importing {model} data'.format(model=model.__name__))
                unflushed_rows = self.table_import(model, row_mapper, table_mapper, unflushed_rows)
            self.destination.session.commit()

        except SQLAlchemyError as e:
            # Rollback the transaction if any error occurs during the transaction
            self.destination.session.rollback()
            # Reraise the exception to prevent other errors occuring due to the non-completion
            raise e

    def end(self):
        self.source.end()
        self.destination.end()


class NoVersionChannelImport(ChannelImport):
    """
    Class defining the schema mapping for importing old content databases (i.e. ones produced before the
    ChannelImport machinery was implemented). The schema mapping below defines how to bring in information
    from the old version of the Kolibri content databases into the database for the current version of Kolibri.
    """

    schema_mapping = {
        # The top level keys of the schema_mapping are the Content Django Models that are to be imported
        ContentNode: {
            # For each model's mappings, can defined both 'per_row' and 'per_table' mappings.
            'per_row': {
                # The key of the 'per_row' mapping object is the table column that we are populating
                # In the case of Django ForeignKey fields, this will be the field name plus _id
                # The value is a string that refers either to a table column on the source data
                # or a method on this import class that will be passed the row data and should return
{'fraction_non_alphanumeric': 0.05617240571562427, 'fraction_numerical': 0.0029983602717263995, 'mean_word_length': 2.29159599074788, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1199679', 'n_tokens_mistral': 5551, 'n_tokens_neox': 5279, 'n_words': 2164}