<commit_before>import sys

from django.utils.six.moves import input

from kolibri.core.auth.models import AdHocGroup
from kolibri.core.auth.models import Membership
from kolibri.core.logger.models import AttemptLog
from kolibri.core.logger.models import ContentSessionLog
from kolibri.core.logger.models import ContentSummaryLog
from kolibri.core.logger.models import MasteryLog
from kolibri.core.logger.models import UserSessionLog


def confirm_or_exit(message):
    answer = ""
    while answer not in ["yes", "n", "no"]:
        answer = input("{} [Type 'yes' or 'no'.] ".format(message)).lower()
    if answer != "yes":
        print("Canceled! Exiting without touching the database.")
        sys.exit(1)


def create_adhoc_group_for_learners(classroom, learners):
    adhoc_group = AdHocGroup.objects.create(name="", parent=classroom)
    for learner in learners:
        Membership.objects.create(user=learner, collection=adhoc_group)
    return adhoc_group


models_to_copy = [
    ContentSessionLog,
    ContentSummaryLog,
    MasteryLog,
    AttemptLog,
    UserSessionLog,
]

user_fields = [
    "gender",
    "birth_year",
    "id_number",
]

log_fields = {
    ContentSessionLog: [
        "content_id",
        "visitor_id",
        "channel_id",
        "start_timestamp",
        "end_timestamp",
        "time_spent",
        "progress",
        "kind",
        "extra_fields",
    ],
    ContentSummaryLog: [
        "content_id",
        "channel_id",
        "start_timestamp",
        "end_timestamp",
        "completion_timestamp",
        "time_spent",
        "progress",
        "kind",
        "extra_fields",
    ],
    UserSessionLog: [
        "channels",
        "start_timestamp",
        "last_interaction_timestamp",
        "pages",
        "device_info",
    ],
    MasteryLog: [
        "mastery_criterion",
        "start_timestamp",
        "end_timestamp",
        "completion_timestamp",
        "mastery_level",
        "complete",
        ("summarylog_id", ContentSummaryLog),
    ],
    AttemptLog: [
        "item",
        "start_timestamp",
        "end_timestamp",
        "completion_timestamp",
        "time_spent",
        "complete",
        "correct",
        "hinted",
        "answer",
        "simple_answer",
        "interaction_history",
        "error",
        ("masterylog_id", MasteryLog),
        ("sessionlog_id", ContentSessionLog),
    ],
}


def _merge_user_models(source_user, target_user):
    for f in user_fields:
        source_value = getattr(source_user, f, None)
        target_value = getattr(target_user, f, None)
        if not target_value and source_value:
            setattr(target_user, f, source_value)
    target_user.save()


def merge_users(source_user, target_user):
    """
    Utility to merge two users. It makes no assumptions about whether
    the users are in the same facility and does raw copies of all
    associated user data, rather than try to do anything clever.
    """
    if source_user.id == target_user.id:
        raise ValueError("Cannot merge a user with themselves")

    _merge_user_models(source_user, target_user)

    id_map = {}

    def _merge_log_data(source_user, target_user, LogModel):
        log_map = {}
        id_map[LogModel] = log_map
        for log in LogModel.objects.filter(user=source_user):
            data = {}
            for f in log_fields[LogModel]:
                if isinstance(f, tuple):
                    field_name, model = f
                    data[field_name] = id_map[model][getattr(log, field_name)]
                else:
                    data[f] = getattr(log, f)
            new_log = LogModel.objects.create(user=target_user, **data)
            log_map[log.id] = new_log.id

    _merge_log_data(source_user, target_user, ContentSessionLog)

    _merge_log_data(source_user, target_user, ContentSummaryLog)

    _merge_log_data(source_user, target_user, UserSessionLog)

    _merge_log_data(source_user, target_user, MasteryLog)

    _merge_log_data(source_user, target_user, AttemptLog)

    source_user.delete()<commit_msg>Optimize user merging with bulk create and dataset caching.<commit_after>import sys

from django.utils.six.moves import input

from kolibri.core.auth.models import AdHocGroup
from kolibri.core.auth.models import dataset_cache
from kolibri.core.auth.models import Membership
from kolibri.core.logger.models import AttemptLog
from kolibri.core.logger.models import ContentSessionLog
from kolibri.core.logger.models import ContentSummaryLog
from kolibri.core.logger.models import MasteryLog
from kolibri.core.logger.models import UserSessionLog


def confirm_or_exit(message):
    answer = ""
    while answer not in ["yes", "n", "no"]:
        answer = input("{} [Type 'yes' or 'no'.] ".format(message)).lower()
    if answer != "yes":
        print("Canceled! Exiting without touching the database.")
        sys.exit(1)


def create_adhoc_group_for_learners(classroom, learners):
    adhoc_group = AdHocGroup.objects.create(name="", parent=classroom)
    for learner in learners:
        Membership.objects.create(user=learner, collection=adhoc_group)
    return adhoc_group


models_to_copy = [
    ContentSessionLog,
    ContentSummaryLog,
    MasteryLog,
    AttemptLog,
    UserSessionLog,
]

user_fields = [
    "gender",
    "birth_year",
    "id_number",
]

log_fields = {
    ContentSessionLog: [
        "content_id",
        "visitor_id",
        "channel_id",
        "start_timestamp",
        "end_timestamp",
        "time_spent",
        "progress",
        "kind",
        "extra_fields",
    ],
    ContentSummaryLog: [
        "content_id",
        "channel_id",
        "start_timestamp",
        "end_timestamp",
        "completion_timestamp",
        "time_spent",
        "progress",
        "kind",
        "extra_fields",
    ],
    UserSessionLog: [
        "channels",
        "start_timestamp",
        "last_interaction_timestamp",
        "pages",
        "device_info",
    ],
    MasteryLog: [
        "mastery_criterion",
        "start_timestamp",
        "end_timestamp",
        "completion_timestamp",
        "mastery_level",
        "complete",
        ("summarylog_id", ContentSummaryLog),
    ],
    AttemptLog: [
        "item",
        "start_timestamp",
        "end_timestamp",
        "completion_timestamp",
        "time_spent",
        "complete",
        "correct",
        "hinted",
        "answer",
        "simple_answer",
        "interaction_history",
        "error",
        ("masterylog_id", MasteryLog),
        ("sessionlog_id", ContentSessionLog),
    ],
}


def _merge_user_models(source_user, target_user):
    for f in user_fields:
        source_value = getattr(source_user, f, None)
        target_value = getattr(target_user, f, None)
        if not target_value and source_value:
            setattr(target_user, f, source_value)
    target_user.save()


def merge_users(source_user, target_user):
    """
    Utility to merge two users. It makes no assumptions about whether
    the users are in the same facility and does raw copies of all
    associated user data, rather than try to do anything clever.
    """
    if source_user.id == target_user.id:
        raise ValueError("Cannot merge a user with themselves")

    dataset_cache.clear()
    dataset_cache.activate()

    _merge_user_models(source_user, target_user)

    id_map = {}

    def _merge_log_data(source_user, target_user, LogModel):
        log_map = {}
        id_map[LogModel] = log_map
        new_logs = []
        for log in LogModel.objects.filter(user=source_user):
            data = {}
            for f in log_fields[LogModel]:
                if isinstance(f, tuple):
                    field_name, model = f
                    data[field_name] = id_map[model][getattr(log, field_name)]
                else:
                    data[f] = getattr(log, f)
            new_log = LogModel(user=target_user, **data)
            new_log.id = new_log.calculate_uuid()
            new_log.ensure_dataset()
            log_map[log.id] = new_log.id
            new_logs.append(new_log)
        LogModel.objects.bulk_create(new_logs, batch_size=750)

    _merge_log_data(source_user, target_user, ContentSessionLog)

    _merge_log_data(source_user, target_user, ContentSummaryLog)

    _merge_log_data(source_user, target_user, UserSessionLog)

    _merge_log_data(source_user, target_user, MasteryLog)

    _merge_log_data(source_user, target_user, AttemptLog)

    source_user.delete()

    dataset_cache.deactivate()
{'fraction_non_alphanumeric': 0.10141537021873903, 'fraction_numerical': 0.0005848637267516669, 'mean_word_length': 2.6305732484076434, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7526924', 'n_tokens_mistral': 2643, 'n_tokens_neox': 2573, 'n_words': 545}