<commit_before>import collections
import copy
import functools
import itertools
import json
import time
import warnings
from sentinels import NOTHING
from .filtering import filter_applies, resolve_key_value
from . import ObjectId, OperationFailure
from .helpers import basestring

try:
    # Optional requirements for providing Map-Reduce functionality
    import execjs
except ImportError:
    execjs = None

try:
    from bson import (json_util, SON)
except ImportError:
    json_utils = SON = None

from six import (
                 iteritems,
                 itervalues,
                 iterkeys)
from mongomock import helpers


class Collection(object):
    def __init__(self, db, name):
        super(Collection, self).__init__()
        self.name = name
        self._Collection__database = db
        self._documents = {}

    def __repr__(self):
        return "Collection({0}, '{1}')".format(self._Collection__database, self.name)

    def insert(self, data):
        if isinstance(data, list):
            return [self._insert(element) for element in data]
        return self._insert(data)

    def _insert(self, data):
        if '_id' not in data:
            data['_id'] = ObjectId()
        object_id = data['_id']
        if object_id in self._documents:
            raise DuplicateKeyError("Duplicate Key Error", 11000)
        self._documents[object_id] = self._internalize_dict(data)
        return object_id

    def _internalize_dict(self, d):
        return dict((k, copy.deepcopy(v)) for k, v in iteritems(d))

    def _has_key(self, doc, key):
        return key in doc

    def update(self, spec, document, upsert = False, manipulate = False,
               safe = False, multi = False, _check_keys = False, **kwargs):
        """Updates document(s) in the collection."""
        found = False
        updated_existing = False
        num_updated = 0
        for existing_document in itertools.chain(self._iter_documents(spec), [None]):
            # the sentinel document means we should do an upsert
            if existing_document is None:
                if not upsert:
                    continue
                existing_document = self._documents[self._insert(self._discard_operators(spec))]
            else:
                updated_existing = True
            num_updated += 1
            first = True
            found = True
            subdocument = None
            for k, v in iteritems(document):
                if k == '$set':
                    self._update_document_fields(existing_document, v, _set_updater)
                elif k == '$unset':
                    for field, value in iteritems(v):
                        if self._has_key(existing_document, field):
                            del existing_document[field]
                elif k == '$inc':
                    positional = False
                    for key in iterkeys(v):
                        if '$' in key:
                            positional = True
                            break

                    if positional:
                        subdocument = self._update_document_fields_positional(existing_document, v, spec, _inc_updater, subdocument)
                        continue
                    self._update_document_fields(existing_document, v, _inc_updater)
                elif k == '$addToSet':
                    for field, value in iteritems(v):
                        container = existing_document.setdefault(field, [])
                        if value not in container:
                            container.append(value)
                elif k == '$pull':
                    for field, value in iteritems(v):
                        nested_field_list = field.rsplit('.')
                        if len(nested_field_list) == 1:
                            arr = existing_document[field]
                            existing_document[field] = [obj for obj in arr if not obj == value]
                            continue

                        # nested fields includes a positional element
                        # need to find that element
                        if '$' in nested_field_list:
                            if not subdocument:
                                subdocument = self._get_subdocument(existing_document, spec, nested_field_list)

                            # value should be a dictionary since we're pulling
                            pull_results = []
                            # and the last subdoc should be an array
                            for obj in subdocument[nested_field_list[-1]]:
                                if isinstance(obj, dict):
                                    for pull_key, pull_value in iteritems(value):
                                        if obj[pull_key] != pull_value:
                                            pull_results.append(obj)
                                    continue
                                if obj != value:
                                    pull_results.append(obj)

                            # cannot write to doc directly as it doesn't save to existing_document
                            subdocument[nested_field_list[-1]] = pull_results
                elif k == '$push':
                    for field, value in iteritems(v):
                        nested_field_list = field.rsplit('.')
                        if len(nested_field_list) == 1:
                            # document should be a list
                            # append to it
                            if isinstance(value, dict):
                                if '$each' in value:
                                    # append the list to the field
                                    existing_document[field] += list(value['$each'])
                                    continue
                            existing_document[field].append(value)
                            continue

                        # nested fields includes a positional element
                        # need to find that element
                        if '$' in nested_field_list:
                            if not subdocument:
                                subdocument = self._get_subdocument(existing_document, spec, nested_field_list)

                            # we're pushing a list
                            push_results = []
                            if nested_field_list[-1] in subdocument:
                                # if the list exists, then use that list
                                push_results = subdocument[nested_field_list[-1]]

                            if isinstance(value, dict):
                                # check to see if we have the format
                                # { '$each': [] }
                                if '$each' in value:
                                    push_results += list(value['$each'])
                                else:
                                    push_results.append(value)
                            else:
                                push_results.append(value)

                            # cannot write to doc directly as it doesn't save to existing_document
                            subdocument[nested_field_list[-1]] = push_results
                else:
                    if first:
                        # replace entire document
                        for key in document.keys():
                            if key.startswith('$'):
                                # can't mix modifiers with non-modifiers in update
                                raise ValueError('field names cannot start with $ [{}]'.format(k))
                        _id = spec.get('_id', existing_document.get('_id', None))
                        existing_document.clear()
                        if _id:
                            existing_document['_id'] = _id
                        existing_document.update(document)
                        if existing_document['_id'] != _id:
                            # id changed, fix index
                            del self._documents[_id]
                            self.insert(existing_document)
                        break
                    else:
                        # can't mix modifiers with non-modifiers in update
                        raise ValueError('Invalid modifier specified: {}'.format(k))
                first = False
            if not multi:
                break

        return {
            "connectionId": self._Collection__database.connection._id,
            "err": None,
            "ok": 1.0,
            "n": num_updated,
            "updateExisting": updated_existing,
        }

    def _get_subdocument(self, existing_document, spec, nested_field_list):
        """
        This method retrieves the subdocument of the existing_document.nested_field_list. It uses the spec to filter
        through the items. It will continue to grab nested documents until it can go no further. It will then return the
        subdocument that was last saved. '$' is the positional operator, so we use the $elemMatch in the spec to find
        the right subdocument in the array.
        """
        # current document in view
        doc = existing_document
        # previous document in view
        subdocument = existing_document
        # current spec in view
        subspec = spec
        # walk down the dictionary
        for subfield in nested_field_list:
            if subfield == '$':
                # positional element should have the equivalent elemMatch in the query
                subspec = subspec['$elemMatch']
                for item in doc:
                    # iterate through
                    if filter_applies(subspec, item):
                        # found the matching item<commit_msg>Fix issue where return keys are supposed to be unicode. Also, fixed typo in updatedExisting key.<commit_after>import collections
import copy
import functools
import itertools
import json
import time
import warnings
from sentinels import NOTHING
from .filtering import filter_applies, resolve_key_value
from . import ObjectId, OperationFailure
from .helpers import basestring

try:
    # Optional requirements for providing Map-Reduce functionality
    import execjs
except ImportError:
    execjs = None

try:
    from bson import (json_util, SON)
except ImportError:
    json_utils = SON = None

from six import (
                 iteritems,
                 itervalues,
                 iterkeys)
from mongomock import helpers


class Collection(object):
    def __init__(self, db, name):
        super(Collection, self).__init__()
        self.name = name
        self._Collection__database = db
        self._documents = {}

    def __repr__(self):
        return "Collection({0}, '{1}')".format(self._Collection__database, self.name)

    def insert(self, data):
        if isinstance(data, list):
            return [self._insert(element) for element in data]
        return self._insert(data)

    def _insert(self, data):
        if '_id' not in data:
            data['_id'] = ObjectId()
        object_id = data['_id']
        if object_id in self._documents:
            raise DuplicateKeyError("Duplicate Key Error", 11000)
        self._documents[object_id] = self._internalize_dict(data)
        return object_id

    def _internalize_dict(self, d):
        return dict((k, copy.deepcopy(v)) for k, v in iteritems(d))

    def _has_key(self, doc, key):
        return key in doc

    def update(self, spec, document, upsert = False, manipulate = False,
               safe = False, multi = False, _check_keys = False, **kwargs):
        """Updates document(s) in the collection."""
        found = False
        updated_existing = False
        num_updated = 0
        for existing_document in itertools.chain(self._iter_documents(spec), [None]):
            # the sentinel document means we should do an upsert
            if existing_document is None:
                if not upsert:
                    continue
                existing_document = self._documents[self._insert(self._discard_operators(spec))]
            else:
                updated_existing = True
            num_updated += 1
            first = True
            found = True
            subdocument = None
            for k, v in iteritems(document):
                if k == '$set':
                    self._update_document_fields(existing_document, v, _set_updater)
                elif k == '$unset':
                    for field, value in iteritems(v):
                        if self._has_key(existing_document, field):
                            del existing_document[field]
                elif k == '$inc':
                    positional = False
                    for key in iterkeys(v):
                        if '$' in key:
                            positional = True
                            break

                    if positional:
                        subdocument = self._update_document_fields_positional(existing_document, v, spec, _inc_updater, subdocument)
                        continue
                    self._update_document_fields(existing_document, v, _inc_updater)
                elif k == '$addToSet':
                    for field, value in iteritems(v):
                        container = existing_document.setdefault(field, [])
                        if value not in container:
                            container.append(value)
                elif k == '$pull':
                    for field, value in iteritems(v):
                        nested_field_list = field.rsplit('.')
                        if len(nested_field_list) == 1:
                            arr = existing_document[field]
                            existing_document[field] = [obj for obj in arr if not obj == value]
                            continue

                        # nested fields includes a positional element
                        # need to find that element
                        if '$' in nested_field_list:
                            if not subdocument:
                                subdocument = self._get_subdocument(existing_document, spec, nested_field_list)

                            # value should be a dictionary since we're pulling
                            pull_results = []
                            # and the last subdoc should be an array
                            for obj in subdocument[nested_field_list[-1]]:
                                if isinstance(obj, dict):
                                    for pull_key, pull_value in iteritems(value):
                                        if obj[pull_key] != pull_value:
                                            pull_results.append(obj)
                                    continue
                                if obj != value:
                                    pull_results.append(obj)

                            # cannot write to doc directly as it doesn't save to existing_document
                            subdocument[nested_field_list[-1]] = pull_results
                elif k == '$push':
                    for field, value in iteritems(v):
                        nested_field_list = field.rsplit('.')
                        if len(nested_field_list) == 1:
                            # document should be a list
                            # append to it
                            if isinstance(value, dict):
                                if '$each' in value:
                                    # append the list to the field
                                    existing_document[field] += list(value['$each'])
                                    continue
                            existing_document[field].append(value)
                            continue

                        # nested fields includes a positional element
                        # need to find that element
                        if '$' in nested_field_list:
                            if not subdocument:
                                subdocument = self._get_subdocument(existing_document, spec, nested_field_list)

                            # we're pushing a list
                            push_results = []
                            if nested_field_list[-1] in subdocument:
                                # if the list exists, then use that list
                                push_results = subdocument[nested_field_list[-1]]

                            if isinstance(value, dict):
                                # check to see if we have the format
                                # { '$each': [] }
                                if '$each' in value:
                                    push_results += list(value['$each'])
                                else:
                                    push_results.append(value)
                            else:
                                push_results.append(value)

                            # cannot write to doc directly as it doesn't save to existing_document
                            subdocument[nested_field_list[-1]] = push_results
                else:
                    if first:
                        # replace entire document
                        for key in document.keys():
                            if key.startswith('$'):
                                # can't mix modifiers with non-modifiers in update
                                raise ValueError('field names cannot start with $ [{}]'.format(k))
                        _id = spec.get('_id', existing_document.get('_id', None))
                        existing_document.clear()
                        if _id:
                            existing_document['_id'] = _id
                        existing_document.update(document)
                        if existing_document['_id'] != _id:
                            # id changed, fix index
                            del self._documents[_id]
                            self.insert(existing_document)
                        break
                    else:
                        # can't mix modifiers with non-modifiers in update
                        raise ValueError('Invalid modifier specified: {}'.format(k))
                first = False
            if not multi:
                break

        return {
            u"connectionId": self._Collection__database.connection._id,
            u"err": None,
            u"ok": 1.0,
            u"n": num_updated,
            u"updatedExisting": updated_existing,
        }

    def _get_subdocument(self, existing_document, spec, nested_field_list):
        """
        This method retrieves the subdocument of the existing_document.nested_field_list. It uses the spec to filter
        through the items. It will continue to grab nested documents until it can go no further. It will then return the
        subdocument that was last saved. '$' is the positional operator, so we use the $elemMatch in the spec to find
        the right subdocument in the array.
        """
        # current document in view
        doc = existing_document
        # previous document in view
        subdocument = existing_document
        # current spec in view
        subspec = spec
        # walk down the dictionary
        for subfield in nested_field_list:
            if subfield == '$':
                # positional element should have the equivalent elemMatch in the query
                subspec = subspec['$elemMatch']
                for item in doc:
                    # iterate through
                    if filter_applies(subspec, item):
                        # found the matching item
{'fraction_non_alphanumeric': 0.06611696442204917, 'fraction_numerical': 0.0018323408153916628, 'mean_word_length': 1.2439470077661032, 'pattern_counts': {'":': 10, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2814492', 'n_tokens_mistral': 4493, 'n_tokens_neox': 4259, 'n_words': 1374}