<commit_before>import fnmatch
import logging
import os

from sqlalchemy.exc import DatabaseError

from .paths import get_content_database_dir_path
from .sqlalchemybridge import Bridge
from kolibri.core.content.models import ChannelMetadata
from kolibri.core.discovery.utils.filesystem import enumerate_mounted_disk_partitions
from kolibri.utils.uuids import is_valid_uuid

logger = logging.getLogger(__name__)


def get_channel_ids_for_content_database_dir(content_database_dir):
    """
    Returns a list of channel IDs for the channel databases that exist in a content database directory.
    """

    # immediately return an empty list if the content database directory doesn't exist
    if not os.path.isdir(content_database_dir):
        return []

    # get a list of all the database files in the directory, and extract IDs
    db_list = fnmatch.filter(os.listdir(content_database_dir), '*.sqlite3')
    db_names = [db.split('.sqlite3', 1)[0] for db in db_list]

    # determine which database names are valid, and only use those ones
    valid_db_names = [name for name in db_names if is_valid_uuid(name)]
    invalid_db_names = set(db_names) - set(valid_db_names)
    if invalid_db_names:
        logger.warning("Ignoring databases in content database directory '{directory}' with invalid names: {names}"
                       .format(directory=content_database_dir, names=invalid_db_names))

    # nonexistent database files are created if we delete the files that have broken symbolic links;
    # empty database files are created if we delete a database file while the server is running and connected to it;
    # here, we delete and exclude such databases to avoid errors when we try to connect to them
    db_files_to_remove = set({})
    for db_name in valid_db_names:
        filename = os.path.join(content_database_dir, "{}.sqlite3".format(db_name))
        if not os.path.exists(filename) or os.path.getsize(filename) == 0:
            db_files_to_remove.add(db_name)
            os.remove(filename)
            # Delete the channel metadata from the database if exists, so that
            # users can download the database file again from the channel import page.
            channel = ChannelMetadata.objects.filter(id=db_name)
            if channel:
                channel.delete()
    if db_files_to_remove:
        err_msg = (
            "Removing nonexistent or empty databases in content database directory "
            "'{directory}' with IDs: {names}.\nPlease import the channels again."
        )
        logger.warning(
            err_msg.format(directory=content_database_dir, names=db_files_to_remove)
        )
    valid_dbs = list(set(valid_db_names) - set(db_files_to_remove))

    return valid_dbs


def enumerate_content_database_file_paths(content_database_dir):
    full_dir_template = os.path.join(content_database_dir, "{}.sqlite3")
    channel_ids = get_channel_ids_for_content_database_dir(content_database_dir)
    return [full_dir_template.format(f) for f in channel_ids]
<commit_msg>do not delete the channel metadata from the database<commit_after>import fnmatch
import logging
import os

from sqlalchemy.exc import DatabaseError

from .paths import get_content_database_dir_path
from .sqlalchemybridge import Bridge
from kolibri.core.discovery.utils.filesystem import enumerate_mounted_disk_partitions
from kolibri.utils.uuids import is_valid_uuid

logger = logging.getLogger(__name__)


def get_channel_ids_for_content_database_dir(content_database_dir):
    """
    Returns a list of channel IDs for the channel databases that exist in a content database directory.
    """

    # immediately return an empty list if the content database directory doesn't exist
    if not os.path.isdir(content_database_dir):
        return []

    # get a list of all the database files in the directory, and extract IDs
    db_list = fnmatch.filter(os.listdir(content_database_dir), '*.sqlite3')
    db_names = [db.split('.sqlite3', 1)[0] for db in db_list]

    # determine which database names are valid, and only use those ones
    valid_db_names = [name for name in db_names if is_valid_uuid(name)]
    invalid_db_names = set(db_names) - set(valid_db_names)
    if invalid_db_names:
        logger.warning("Ignoring databases in content database directory '{directory}' with invalid names: {names}"
                       .format(directory=content_database_dir, names=invalid_db_names))

    # nonexistent database files are created if we delete the files that have broken symbolic links;
    # empty database files are created if we delete a database file while the server is running and connected to it;
    # here, we delete and exclude such databases to avoid errors when we try to connect to them
    db_files_to_remove = set({})
    for db_name in valid_db_names:
        filename = os.path.join(content_database_dir, "{}.sqlite3".format(db_name))
        if not os.path.exists(filename) or os.path.getsize(filename) == 0:
            db_files_to_remove.add(db_name)
            os.remove(filename)

    if db_files_to_remove:
        err_msg = (
            "Removing nonexistent or empty databases in content database directory "
            "'{directory}' with IDs: {names}.\nPlease import the channels again."
        )
        logger.warning(
            err_msg.format(directory=content_database_dir, names=db_files_to_remove)
        )
    valid_dbs = list(set(valid_db_names) - set(db_files_to_remove))

    return valid_dbs


def enumerate_content_database_file_paths(content_database_dir):
    full_dir_template = os.path.join(content_database_dir, "{}.sqlite3")
    channel_ids = get_channel_ids_for_content_database_dir(content_database_dir)
    return [full_dir_template.format(f) for f in channel_ids]

{'fraction_non_alphanumeric': 0.06984402079722704, 'fraction_numerical': 0.0024263431542461003, 'mean_word_length': 3.7225859247135844, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2873665', 'n_tokens_mistral': 1664, 'n_tokens_neox': 1605, 'n_words': 508}