<commit_before>import pymongo
from bson.objectid import ObjectId
from importlib import import_module
from xmodule.x_module import XModuleDescriptor
from xmodule.mako_module import MakoDescriptorSystem
from mitxmako.shortcuts import render_to_string
from bson.son import SON
from itertools import repeat

from . import ModuleStore, Location
from .exceptions import ItemNotFoundError, InsufficientSpecificationError

# TODO (cpennington): This code currently operates under the assumption that
# there is only one revision for each item. Once we start versioning inside the CMS,
# that assumption will have to change


class CachingDescriptorSystem(MakoDescriptorSystem):
    """
    A system that has a cache of module json that it will use to load modules
    from, with a backup of calling to the underlying modulestore for more data
    """
    def __init__(self, modulestore, module_data, default_class, resources_fs, render_template):
        """
        modulestore: the module store that can be used to retrieve additional modules
        module_data: a mapping of Location -> json that was cached from the underlying modulestore
        default_class: The default_class to use when loading an XModuleDescriptor from the module_data
        resources_fs: a filesystem, as per MakoDescriptorSystem
        render_template: a function for rendering templates, as per MakoDescriptorSystem
        """
        super(CachingDescriptorSystem, self).__init__(render_template, self.load_item, resources_fs)
        self.modulestore = modulestore
        self.module_data = module_data
        self.default_class = default_class

    def load_item(self, location):
        location = Location(location)
        json_data = self.module_data.get(location)
        if json_data is None:
            return self.modulestore.get_item(location)
        else:
            return XModuleDescriptor.load_from_json(json_data, self, self.default_class)


def location_to_query(loc):
    """
    Takes a Location and returns a SON object that will query for that location.
    Fields in loc that are None are ignored in the query
    """
    query = SON()
    # Location dict is ordered by specificity, and SON
    # will preserve that order for queries
    for key, val in Location(loc).dict().iteritems():
        if val is not None:
            query['_id.{key}'.format(key=key)] = val

    return query

class MongoModuleStore(ModuleStore):
    """
    A Mongodb backed ModuleStore
    """
    def __init__(self, host, db, collection, port=27017, default_class=None):
        self.collection = pymongo.connection.Connection(
            host=host,
            port=port
        )[db][collection]

        # Force mongo to report errors, at the expense of performance
        self.collection.safe = True

        # Force mongo to maintain an index over _id.* that is in the same order
        # that is used when querying by a location
        self.collection.ensure_index(zip(('_id.' + field for field in Location._fields), repeat(1)))

        module_path, _, class_name = default_class.rpartition('.')
        class_ = getattr(import_module(module_path), class_name)
        self.default_class = class_

    def _clean_item_data(self, item):
        """
        Renames the '_id' field in item to 'location'
        """
        item['location'] = item['_id']
        del item['_id']

    def _cache_children(self, items, depth=0):
        """
        Returns a dictionary mapping Location -> item data, populated with json data
        for all descendents of items up to the specified depth.
        This will make a number of queries that is linear in the depth
        """
        data = {}
        to_process = list(items)
        while to_process and depth is None or depth >= 0:
            children = []
            for item in to_process:
                self._clean_item_data(item)
                children.extend(item.get('definition', {}).get('children', []))
                data[Location(item['location'])] = item

            # Load all children by id. See
            # http://www.mongodb.org/display/DOCS/Advanced+Queries#AdvancedQueries-%24or
            # for or-query syntax
            if children:
                to_process = list(self.collection.find({'_id': {'$in': [Location(child).dict() for child in children]}}))
            else:
                to_process = []
            if depth is not None:
                depth -= 1

        return data

    def _load_items(self, items, depth=0):
        """
        Load a list of xmodules from the data in items, with children cached up to specified depth
        """
        data_cache = self._cache_children(items, depth)
        system = CachingDescriptorSystem(self, data_cache, self.default_class, None, render_to_string)
        return [system.load_item(item['location']) for item in items]

    def get_item(self, location, depth=0):
        """
        Returns an XModuleDescriptor instance for the item at location.
        If location.revision is None, returns the most item with the most
        recent revision

        If any segment of the location is None except revision, raises
            xmodule.modulestore.exceptions.InsufficientSpecificationError
        If no object is found at that location, raises xmodule.modulestore.exceptions.ItemNotFoundError

        location: a Location object
        depth (int): An argument that some module stores may use to prefetch descendents of the queried modules<commit_msg>Minor changes from pull request 170<commit_after>import pymongo
from bson.objectid import ObjectId
from importlib import import_module
from xmodule.x_module import XModuleDescriptor
from xmodule.mako_module import MakoDescriptorSystem
from mitxmako.shortcuts import render_to_string
from bson.son import SON
from itertools import repeat

from . import ModuleStore, Location
from .exceptions import ItemNotFoundError, InsufficientSpecificationError

# TODO (cpennington): This code currently operates under the assumption that
# there is only one revision for each item. Once we start versioning inside the CMS,
# that assumption will have to change


class CachingDescriptorSystem(MakoDescriptorSystem):
    """
    A system that has a cache of module json that it will use to load modules
    from, with a backup of calling to the underlying modulestore for more data
    """
    def __init__(self, modulestore, module_data, default_class, resources_fs, render_template):
        """
        modulestore: the module store that can be used to retrieve additional modules
        module_data: a dict mapping Location -> json that was cached from the underlying modulestore
        default_class: The default_class to use when loading an XModuleDescriptor from the module_data
        resources_fs: a filesystem, as per MakoDescriptorSystem
        render_template: a function for rendering templates, as per MakoDescriptorSystem
        """
        super(CachingDescriptorSystem, self).__init__(render_template, self.load_item, resources_fs)
        self.modulestore = modulestore
        self.module_data = module_data
        self.default_class = default_class

    def load_item(self, location):
        location = Location(location)
        json_data = self.module_data.get(location)
        if json_data is None:
            return self.modulestore.get_item(location)
        else:
            return XModuleDescriptor.load_from_json(json_data, self, self.default_class)


def location_to_query(location):
    """
    Takes a Location and returns a SON object that will query for that location.
    Fields in location that are None are ignored in the query
    """
    query = SON()
    # Location dict is ordered by specificity, and SON
    # will preserve that order for queries
    for key, val in Location(location).dict().iteritems():
        if val is not None:
            query['_id.{key}'.format(key=key)] = val

    return query

class MongoModuleStore(ModuleStore):
    """
    A Mongodb backed ModuleStore
    """
    def __init__(self, host, db, collection, port=27017, default_class=None):
        self.collection = pymongo.connection.Connection(
            host=host,
            port=port
        )[db][collection]

        # Force mongo to report errors, at the expense of performance
        self.collection.safe = True

        # Force mongo to maintain an index over _id.* that is in the same order
        # that is used when querying by a location
        self.collection.ensure_index(zip(('_id.' + field for field in Location._fields), repeat(1)))

        module_path, _, class_name = default_class.rpartition('.')
        class_ = getattr(import_module(module_path), class_name)
        self.default_class = class_

    def _clean_item_data(self, item):
        """
        Renames the '_id' field in item to 'location'
        """
        item['location'] = item['_id']
        del item['_id']

    def _cache_children(self, items, depth=0):
        """
        Returns a dictionary mapping Location -> item data, populated with json data
        for all descendents of items up to the specified depth.
        This will make a number of queries that is linear in the depth
        """
        data = {}
        to_process = list(items)
        while to_process and depth is None or depth >= 0:
            children = []
            for item in to_process:
                self._clean_item_data(item)
                children.extend(item.get('definition', {}).get('children', []))
                data[Location(item['location'])] = item

            # Load all children by id. See
            # http://www.mongodb.org/display/DOCS/Advanced+Queries#AdvancedQueries-%24or
            # for or-query syntax
            if children:
                to_process = list(self.collection.find({'_id': {'$in': [Location(child).dict() for child in children]}}))
            else:
                to_process = []
            # If depth is None, then we just recurse until we hit all the descendents
            if depth is not None:
                depth -= 1

        return data

    def _load_items(self, items, depth=0):
        """
        Load a list of xmodules from the data in items, with children cached up to specified depth
        """
        data_cache = self._cache_children(items, depth)
        system = CachingDescriptorSystem(self, data_cache, self.default_class, None, render_to_string)
        return [system.load_item(item['location']) for item in items]

    def get_item(self, location, depth=0):
        """
        Returns an XModuleDescriptor instance for the item at location.
        If location.revision is None, returns the most item with the most
        recent revision

        If any segment of the location is None except revision, raises
            xmodule.modulestore.exceptions.InsufficientSpecificationError
        If no object is found at that location, raises xmodule.modulestore.exceptions.ItemNotFoundError

        location: a Location object
        depth (int): An argument that some module stores may use to prefetch descendents of the queried modules
{'fraction_non_alphanumeric': 0.07233468286099864, 'fraction_numerical': 0.0026090868196131352, 'mean_word_length': 2.7719714964370548, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5099644', 'n_tokens_mistral': 2924, 'n_tokens_neox': 2838, 'n_words': 1064}