<commit_before>"""
Classes to provide the LMS runtime data storage to XBlocks
"""

import json
from collections import defaultdict
from itertools import chain
from .models import (
    StudentModule,
    XModuleUserStateSummaryField,
    XModuleStudentPrefsField,
    XModuleStudentInfoField
)
import logging
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.block_types import BlockTypeKeyV1
from opaque_keys.edx.asides import AsideUsageKeyV1

from django.db import DatabaseError

from xblock.runtime import KeyValueStore
from xblock.exceptions import KeyValueMultiSaveError, InvalidScopeError
from xblock.fields import Scope, UserScope
from xmodule.modulestore.django import modulestore
from xblock.core import XBlockAside

log = logging.getLogger(__name__)


class InvalidWriteError(Exception):
    """
    Raised to indicate that writing to a particular key
    in the KeyValueStore is disabled
    """


def chunks(items, chunk_size):
    """
    Yields the values from items in chunks of size chunk_size
    """
    items = list(items)
    return (items[i:i + chunk_size] for i in xrange(0, len(items), chunk_size))


def _query(model_class, select_for_update, **kwargs):
    """
    Queries model_class with **kwargs, optionally adding select_for_update if
    `select_for_update` is True.
    """
    query = model_class.objects
    if select_for_update:
        query = query.select_for_update()
    query = query.filter(**kwargs)
    return query

def _chunked_query(model_class, select_for_update, chunk_field, items, chunk_size=500, **kwargs):
    """
    Queries model_class with `chunk_field` set to chunks of size `chunk_size`,
    and all other parameters from `**kwargs`.

    This works around a limitation in sqlite3 on the number of parameters
    that can be put into a single query.
    """
    res = chain.from_iterable(
        _query(model_class, select_for_update, **dict([(chunk_field, chunk)] + kwargs.items()))
        for chunk in chunks(items, chunk_size)
    )
    return res


class UserStateCache(object):
    """
    Cache for Scope.user_state xblock field data.
    """
    def __init__(self, user, course_id, usage_keys, select_for_update=False):
        self._data = _chunked_query(
            StudentModule,
            select_for_update,
            'module_state_key__in',
            usage_keys,
            course_id=course_id,
            student=user.pk,
        )


class UserStateSummaryCache(object):
    """
    Cache for Scope.user_state_summary xblock field data.
    """
    def __init__(self, usage_keys, fields, select_for_update=False):
        self._data = _chunked_query(
            XModuleUserStateSummaryField,
            select_for_update,
            'usage_id__in',
            usage_keys,
            field_name__in=set(field.name for field in fields),
        )


class PreferencesCache(object):
    """
    Cache for Scope.preferences xblock field data.
    """
    def __init__(self, user, block_types, fields, select_for_update=False):
        self._data = _chunked_query(
            XModuleStudentPrefsField,
            select_for_update,
            'module_type__in',
            block_types,
            student=user.pk,
            field_name__in=set(field.name for field in fields),
        )


class UserInfoCache(object):
    """
    Cache for Scope.user_info xblock field data
    """
    def __init__(self, user, fields, select_for_update=False):
        self._data = _query(
            XModuleStudentInfoField,
            select_for_update,
            student=user.pk,
            field_name__in=set(field.name for field in fields),
        )


class FieldDataCache(object):
    """
    A cache of django model objects needed to supply the data
    for a module and its decendants
    """
    def __init__(self, descriptors, course_id, user, select_for_update=False, asides=None):
        '''
        Find any courseware.models objects that are needed by any descriptor
        in descriptors. Attempts to minimize the number of queries to the database.
        Note: Only modules that have store_state = True or have shared
        state will have a StudentModule.

        Arguments
        descriptors: A list of XModuleDescriptors.
        course_id: The id of the current course
        user: The user for which to cache data
        select_for_update: True if rows should be locked until end of transaction
        asides: The list of aside types to load, or None to prefetch no asides.
        '''
        self.cache = {}
        self.select_for_update = select_for_update

        if asides is None:
            self.asides = []
        else:
            self.asides = asides

        assert isinstance(course_id, CourseKey)
        self.course_id = course_id
        self.user = user

        self.add_descriptors_to_cache(descriptors)

    def add_descriptors_to_cache(self, descriptors):
        """
        Add all `descriptors` to this FieldDataCache.
        """
        if self.user.is_authenticated():
            for scope, fields in self._fields_to_cache(descriptors).items():
                self._cache_fields(scope, fields, descriptors)

    def add_descriptor_descendents(self, descriptor, depth=None, descriptor_filter=lambda descriptor: True):
        """
        Add all descendents of `descriptor` to this FieldDataCache.

        Arguments:
            descriptor: An XModuleDescriptor
            depth is the number of levels of descendent modules to load StudentModules for, in addition to
                the supplied descriptor. If depth is None, load all descendent StudentModules
            descriptor_filter is a function that accepts a descriptor and return wether the StudentModule
                should be cached
        """

        def get_child_descriptors(descriptor, depth, descriptor_filter):
            """
            Return a list of all child descriptors down to the specified depth
            that match the descriptor filter. Includes `descriptor`

            descriptor: The parent to search inside
            depth: The number of levels to descend, or None for infinite depth
            descriptor_filter(descriptor): A function that returns True
                if descriptor should be included in the results
            """
            if descriptor_filter(descriptor):
                descriptors = [descriptor]
            else:
                descriptors = []

            if depth is None or depth > 0:
                new_depth = depth - 1 if depth is not None else depth

                for child in descriptor.get_children() + descriptor.get_required_module_descriptors():
                    descriptors.extend(get_child_descriptors(child, new_depth, descriptor_filter))

            return descriptors

        with modulestore().bulk_operations(descriptor.location.course_key):
            descriptors = get_child_descriptors(descriptor, depth, descriptor_filter)

        self.add_descriptors_to_cache(descriptors)

    @classmethod
    def cache_for_descriptor_descendents(cls, course_id, user, descriptor, depth=None,
                                         descriptor_filter=lambda descriptor: True,
                                         select_for_update=False, asides=None):
        """
        course_id: the course in the context of which we want StudentModules.
        user: the django user for whom to load modules.
        descriptor: An XModuleDescriptor
        depth is the number of levels of descendent modules to load StudentModules for, in addition to
            the supplied descriptor. If depth is None, load all descendent StudentModules
        descriptor_filter is a function that accepts a descriptor and return wether the StudentModule
            should be cached
        select_for_update: Flag indicating whether the rows should be locked until end of transaction
        """
        cache = FieldDataCache([], course_id, user, select_for_update, asides=asides)
        cache.add_descriptor_descendents(descriptor, depth, descriptor_filter)
        return cache


    def _all_usage_ids(self, descriptors):
        """
        Return a set of all usage_ids for the descriptors that this FieldDataCache is caching
        against, and well as all asides for those descriptors.
        """
        usage_ids = set()
        for descriptor in descriptors:
            usage_ids.add(descriptor.scope_ids.usage_id)

            for aside_type in self.asides:
                usage_ids.add(AsideUsageKeyV1(descriptor.scope_ids.usage_id, aside_type))

        return usage_ids

    def _all_block_types(self, descriptors):
        """
        Return a set of all block_types that are cached by this FieldDataCache.
        """
        block_types = set()
        for descriptor in descriptors:
            block_types.add(BlockTypeKeyV1(descriptor.entry_point, descriptor.scope_ids.block_type))

        for aside_type in self.asides:
            block_types.add(BlockTypeKeyV1(XBlockAside.entry_point, aside_type))

        return block_types

    def _cache_fields(self, scope, fields, descriptors):
        """
        Queries the database for all of the fields in the specified scope
        """
        if scope == Scope.user_state:
            self.user_state_cache = UserStateCache(
                self.user,
                self.course_id,
                self._all_usage_ids(descriptors),
                self.select_for_update,
            )
            field_objects = self.user_state_cache._data
        elif scope == Scope.user_state_summary:
            self.user_state_summary_cache = UserStateSummaryCache(
                self._all_usage_ids(descriptors),
                fields,
                self.select_for_update,
            )
            field_objects = self.user_state_summary_cache._data
        elif scope == Scope.preferences:
            self.preferences_cache = PreferencesCache(
                self.user,
                self._all_block_types(descriptors),
                fields,
                self.select_for_update,
            )
            field_objects = self.preferences_cache._data
        elif scope == Scope.user_info:
            self.user_info_cache = UserInfoCache(
                self.user,
                fields,
                self.select_for_update,
            )
            field_objects = self.user_info_cache._data
        else:
            field_objects = []

        for field_object in field_objects:
            self.cache[self._cache_key_from_field_object(scope, field_object)] = field_object

    def _fields_to_cache(self, descriptors):
        """
        Returns a map of scopes to fields in that scope that should be cached
        """
        scope_map = defaultdict(set)
        for descriptor in descriptors:
            for field in descriptor.fields.values():
                scope_map[field.scope].add(field)
        return scope_map

    def _cache_key_from_kvs_key(self, key):
        """
        Return the key used in the FieldDataCache for the specified KeyValueStore key
        """
        if key.scope == Scope.user_state:
            return (key.scope, key.block_scope_id)
        elif key.scope == Scope.user_state_summary:
            return (key.scope, key.block_scope_id, key.field_name)
        elif key.scope == Scope.preferences:
            return (key.scope, BlockTypeKeyV1(key.block_family, key.block_scope_id), key.field_name)
        elif key.scope == Scope.user_info:
            return (key.scope, key.field_name)

    def _cache_key_from_field_object(self, scope, field_object):
        """
        Return the key used in the FieldDataCache for the specified scope and
        field
        """
        if scope == Scope.user_state:
            return (scope, field_object.module_state_key.map_into_course(self.course_id))
        elif scope == Scope.user_state_summary:
            return (scope, field_object.usage_id.map_into_course(self.course_id), field_object.field_name)
        elif scope == Scope.preferences:
            return (scope, field_object.module_type, field_object.field_name)
        elif scope == Scope.user_info:
            return (scope, field_object.field_name)
<commit_msg>Move field_object -> cache_key transformations to the scope-specific caches<commit_after>"""
Classes to provide the LMS runtime data storage to XBlocks
"""

import json
from collections import defaultdict
from itertools import chain
from .models import (
    StudentModule,
    XModuleUserStateSummaryField,
    XModuleStudentPrefsField,
    XModuleStudentInfoField
)
import logging
from opaque_keys.edx.keys import CourseKey
from opaque_keys.edx.block_types import BlockTypeKeyV1
from opaque_keys.edx.asides import AsideUsageKeyV1

from django.db import DatabaseError

from xblock.runtime import KeyValueStore
from xblock.exceptions import KeyValueMultiSaveError, InvalidScopeError
from xblock.fields import Scope, UserScope
from xmodule.modulestore.django import modulestore
from xblock.core import XBlockAside

log = logging.getLogger(__name__)


class InvalidWriteError(Exception):
    """
    Raised to indicate that writing to a particular key
    in the KeyValueStore is disabled
    """


def chunks(items, chunk_size):
    """
    Yields the values from items in chunks of size chunk_size
    """
    items = list(items)
    return (items[i:i + chunk_size] for i in xrange(0, len(items), chunk_size))


def _query(model_class, select_for_update, **kwargs):
    """
    Queries model_class with **kwargs, optionally adding select_for_update if
    `select_for_update` is True.
    """
    query = model_class.objects
    if select_for_update:
        query = query.select_for_update()
    query = query.filter(**kwargs)
    return query

def _chunked_query(model_class, select_for_update, chunk_field, items, chunk_size=500, **kwargs):
    """
    Queries model_class with `chunk_field` set to chunks of size `chunk_size`,
    and all other parameters from `**kwargs`.

    This works around a limitation in sqlite3 on the number of parameters
    that can be put into a single query.
    """
    res = chain.from_iterable(
        _query(model_class, select_for_update, **dict([(chunk_field, chunk)] + kwargs.items()))
        for chunk in chunks(items, chunk_size)
    )
    return res


class UserStateCache(object):
    """
    Cache for Scope.user_state xblock field data.
    """
    def __init__(self, user, course_id, usage_keys, select_for_update=False):
        self._data = _chunked_query(
            StudentModule,
            select_for_update,
            'module_state_key__in',
            usage_keys,
            course_id=course_id,
            student=user.pk,
        )
        self.course_id = course_id

    def cache_key_for_field_object(self, field_object):
        return (Scope.user_state, field_object.module_state_key.map_into_course(self.course_id))


class UserStateSummaryCache(object):
    """
    Cache for Scope.user_state_summary xblock field data.
    """
    def __init__(self, course_id, usage_keys, fields, select_for_update=False):
        self._data = _chunked_query(
            XModuleUserStateSummaryField,
            select_for_update,
            'usage_id__in',
            usage_keys,
            field_name__in=set(field.name for field in fields),
        )
        self.course_id = course_id

    def cache_key_for_field_object(self, field_object):
        return (Scope.user_state_summary, field_object.usage_id.map_into_course(self.course_id), field_object.field_name)


class PreferencesCache(object):
    """
    Cache for Scope.preferences xblock field data.
    """
    def __init__(self, user, block_types, fields, select_for_update=False):
        self._data = _chunked_query(
            XModuleStudentPrefsField,
            select_for_update,
            'module_type__in',
            block_types,
            student=user.pk,
            field_name__in=set(field.name for field in fields),
        )

    def cache_key_for_field_object(self, field_object):
        return (Scope.preferences, field_object.module_type, field_object.field_name)


class UserInfoCache(object):
    """
    Cache for Scope.user_info xblock field data
    """
    def __init__(self, user, fields, select_for_update=False):
        self._data = _query(
            XModuleStudentInfoField,
            select_for_update,
            student=user.pk,
            field_name__in=set(field.name for field in fields),
        )

    def cache_key_for_field_object(self, field_object):
        return (Scope.user_info, field_object.field_name)


class FieldDataCache(object):
    """
    A cache of django model objects needed to supply the data
    for a module and its decendants
    """
    def __init__(self, descriptors, course_id, user, select_for_update=False, asides=None):
        '''
        Find any courseware.models objects that are needed by any descriptor
        in descriptors. Attempts to minimize the number of queries to the database.
        Note: Only modules that have store_state = True or have shared
        state will have a StudentModule.

        Arguments
        descriptors: A list of XModuleDescriptors.
        course_id: The id of the current course
        user: The user for which to cache data
        select_for_update: True if rows should be locked until end of transaction
        asides: The list of aside types to load, or None to prefetch no asides.
        '''
        self.cache = {}
        self.select_for_update = select_for_update

        if asides is None:
            self.asides = []
        else:
            self.asides = asides

        assert isinstance(course_id, CourseKey)
        self.course_id = course_id
        self.user = user

        self.add_descriptors_to_cache(descriptors)

    def add_descriptors_to_cache(self, descriptors):
        """
        Add all `descriptors` to this FieldDataCache.
        """
        if self.user.is_authenticated():
            for scope, fields in self._fields_to_cache(descriptors).items():
                self._cache_fields(scope, fields, descriptors)

    def add_descriptor_descendents(self, descriptor, depth=None, descriptor_filter=lambda descriptor: True):
        """
        Add all descendents of `descriptor` to this FieldDataCache.

        Arguments:
            descriptor: An XModuleDescriptor
            depth is the number of levels of descendent modules to load StudentModules for, in addition to
                the supplied descriptor. If depth is None, load all descendent StudentModules
            descriptor_filter is a function that accepts a descriptor and return wether the StudentModule
                should be cached
        """

        def get_child_descriptors(descriptor, depth, descriptor_filter):
            """
            Return a list of all child descriptors down to the specified depth
            that match the descriptor filter. Includes `descriptor`

            descriptor: The parent to search inside
            depth: The number of levels to descend, or None for infinite depth
            descriptor_filter(descriptor): A function that returns True
                if descriptor should be included in the results
            """
            if descriptor_filter(descriptor):
                descriptors = [descriptor]
            else:
                descriptors = []

            if depth is None or depth > 0:
                new_depth = depth - 1 if depth is not None else depth

                for child in descriptor.get_children() + descriptor.get_required_module_descriptors():
                    descriptors.extend(get_child_descriptors(child, new_depth, descriptor_filter))

            return descriptors

        with modulestore().bulk_operations(descriptor.location.course_key):
            descriptors = get_child_descriptors(descriptor, depth, descriptor_filter)

        self.add_descriptors_to_cache(descriptors)

    @classmethod
    def cache_for_descriptor_descendents(cls, course_id, user, descriptor, depth=None,
                                         descriptor_filter=lambda descriptor: True,
                                         select_for_update=False, asides=None):
        """
        course_id: the course in the context of which we want StudentModules.
        user: the django user for whom to load modules.
        descriptor: An XModuleDescriptor
        depth is the number of levels of descendent modules to load StudentModules for, in addition to
            the supplied descriptor. If depth is None, load all descendent StudentModules
        descriptor_filter is a function that accepts a descriptor and return wether the StudentModule
            should be cached
        select_for_update: Flag indicating whether the rows should be locked until end of transaction
        """
        cache = FieldDataCache([], course_id, user, select_for_update, asides=asides)
        cache.add_descriptor_descendents(descriptor, depth, descriptor_filter)
        return cache


    def _all_usage_ids(self, descriptors):
        """
        Return a set of all usage_ids for the descriptors that this FieldDataCache is caching
        against, and well as all asides for those descriptors.
        """
        usage_ids = set()
        for descriptor in descriptors:
            usage_ids.add(descriptor.scope_ids.usage_id)

            for aside_type in self.asides:
                usage_ids.add(AsideUsageKeyV1(descriptor.scope_ids.usage_id, aside_type))

        return usage_ids

    def _all_block_types(self, descriptors):
        """
        Return a set of all block_types that are cached by this FieldDataCache.
        """
        block_types = set()
        for descriptor in descriptors:
            block_types.add(BlockTypeKeyV1(descriptor.entry_point, descriptor.scope_ids.block_type))

        for aside_type in self.asides:
            block_types.add(BlockTypeKeyV1(XBlockAside.entry_point, aside_type))

        return block_types

    def _cache_fields(self, scope, fields, descriptors):
        """
        Queries the database for all of the fields in the specified scope
        """
        if scope == Scope.user_state:
            cache = self.user_state_cache = UserStateCache(
                self.user,
                self.course_id,
                self._all_usage_ids(descriptors),
                self.select_for_update,
            )
        elif scope == Scope.user_state_summary:
            cache = self.user_state_summary_cache = UserStateSummaryCache(
                self.course_id,
                self._all_usage_ids(descriptors),
                fields,
                self.select_for_update,
            )
        elif scope == Scope.preferences:
            cache = self.preferences_cache = PreferencesCache(
                self.user,
                self._all_block_types(descriptors),
                fields,
                self.select_for_update,
            )
        elif scope == Scope.user_info:
            cache = self.user_info_cache = UserInfoCache(
                self.user,
                fields,
                self.select_for_update,
            )
        else:
            return

        for field_object in cache._data:
            self.cache[cache.cache_key_for_field_object(field_object)] = field_object

    def _fields_to_cache(self, descriptors):
        """
        Returns a map of scopes to fields in that scope that should be cached
        """
        scope_map = defaultdict(set)
        for descriptor in descriptors:
            for field in descriptor.fields.values():
                scope_map[field.scope].add(field)
        return scope_map

    def _cache_key_from_kvs_key(self, key):
        """
        Return the key used in the FieldDataCache for the specified KeyValueStore key
        """
        if key.scope == Scope.user_state:
            return (key.scope, key.block_scope_id)
        elif key.scope == Scope.user_state_summary:
            return (key.scope, key.block_scope_id, key.field_name)
        elif key.scope == Scope.preferences:
            return (key.scope, BlockTypeKeyV1(key.block_family, key.block_scope_id), key.field_name)
        elif key.scope == Scope.user_info:
            return (key.scope, key.field_name)

{'fraction_non_alphanumeric': 0.06699353137489185, 'fraction_numerical': 0.0010712372790573112, 'mean_word_length': 2.433097595473833, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3939513', 'n_tokens_mistral': 6525, 'n_tokens_neox': 6385, 'n_words': 1887}