<commit_before>import datetime
import logging as logger
import os

from django.conf import settings
from le_utils.constants import content_kinds
from sqlalchemy import and_
from sqlalchemy import exists
from sqlalchemy import func
from sqlalchemy import select

from .channels import get_channel_ids_for_content_database_dir
from .paths import get_content_database_file_path
from .paths import get_content_file_name
from .paths import get_content_storage_file_path
from .sqlalchemybridge import Bridge
from kolibri.content.apps import KolibriContentConfig
from kolibri.content.models import ChannelMetadata
from kolibri.content.models import ContentNode
from kolibri.content.models import File
from kolibri.content.models import LocalFile

logging = logger.getLogger(__name__)

CONTENT_APP_NAME = KolibriContentConfig.label

CHUNKSIZE = 10000


def update_channel_metadata():
    """
    If we are potentially moving from a version of Kolibri that did not import its content data,
    scan through the settings.CONTENT_DATABASE_DIR folder for all channel content databases,
    and pull the data from each database if we have not already imported it.
    """
    from .channel_import import import_channel_from_local_db
    channel_ids = get_channel_ids_for_content_database_dir(settings.CONTENT_DATABASE_DIR)
    for channel_id in channel_ids:
        if not ChannelMetadata.objects.filter(id=channel_id).exists():
            import_channel_from_local_db(channel_id)
    fix_multiple_trees_with_id_one()


def fix_multiple_trees_with_id_one():
    # Do a check for improperly imported ContentNode trees
    # These trees have been naively imported, and so there are multiple trees
    # with tree_ids set to 1. Just check the root nodes to reduce the query size.
    tree_id_one_channel_ids = ContentNode.objects.filter(parent=None, tree_id=1).values_list('channel_id', flat=True)
    if len(tree_id_one_channel_ids) > 1:
        logging.warning("Improperly imported channels discovered")
        # There is more than one channel with a tree_id of 1
        # Find which channel has the most content nodes, and then delete and reimport the rest.<commit_msg>Make docstring more accurate.<commit_after>import datetime
import logging as logger
import os

from django.conf import settings
from le_utils.constants import content_kinds
from sqlalchemy import and_
from sqlalchemy import exists
from sqlalchemy import func
from sqlalchemy import select

from .channels import get_channel_ids_for_content_database_dir
from .paths import get_content_database_file_path
from .paths import get_content_file_name
from .paths import get_content_storage_file_path
from .sqlalchemybridge import Bridge
from kolibri.content.apps import KolibriContentConfig
from kolibri.content.models import ChannelMetadata
from kolibri.content.models import ContentNode
from kolibri.content.models import File
from kolibri.content.models import LocalFile

logging = logger.getLogger(__name__)

CONTENT_APP_NAME = KolibriContentConfig.label

CHUNKSIZE = 10000


def update_channel_metadata():
    """
    If we are potentially moving from a version of Kolibri that did not import its content data,
    scan through the settings.CONTENT_DATABASE_DIR folder for all channel content databases,
    and pull the data from each database if we have not already imported it.
    Additionally, fix any potential issues that might be in the current content database from bugs
    in a previous version.
    """
    from .channel_import import import_channel_from_local_db
    channel_ids = get_channel_ids_for_content_database_dir(settings.CONTENT_DATABASE_DIR)
    for channel_id in channel_ids:
        if not ChannelMetadata.objects.filter(id=channel_id).exists():
            import_channel_from_local_db(channel_id)
    fix_multiple_trees_with_id_one()


def fix_multiple_trees_with_id_one():
    # Do a check for improperly imported ContentNode trees
    # These trees have been naively imported, and so there are multiple trees
    # with tree_ids set to 1. Just check the root nodes to reduce the query size.
    tree_id_one_channel_ids = ContentNode.objects.filter(parent=None, tree_id=1).values_list('channel_id', flat=True)
    if len(tree_id_one_channel_ids) > 1:
        logging.warning("Improperly imported channels discovered")
        # There is more than one channel with a tree_id of 1
        # Find which channel has the most content nodes, and then delete and reimport the rest.
{'fraction_non_alphanumeric': 0.043721973094170405, 'fraction_numerical': 0.004035874439461884, 'mean_word_length': 4.862023653088042, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4535793', 'n_tokens_mistral': 1251, 'n_tokens_neox': 1212, 'n_words': 441}