<commit_before>from queryset import QuerySet, QuerySetManager
from queryset import DoesNotExist, MultipleObjectsReturned

import sys
import pymongo


_document_registry = {}

def get_document(name):
    return _document_registry[name]


class ValidationError(Exception):
    pass


class BaseField(object):
    """A base class for fields in a MongoDB document. Instances of this class
    may be added to subclasses of `Document` to define a document's schema.
    """

    # Fields may have _types inserted into indexes by default 
    _index_with_types = True
    
    def __init__(self, db_field=None, name=None, required=False, default=None, 
                 unique=False, unique_with=None, primary_key=False, validation=None,
                 create_default=False, choices=None):
        self.db_field = (db_field or name) if not primary_key else '_id'
        if name:
            import warnings
            msg = "Fields' 'name' attribute deprecated in favour of 'db_field'"
            warnings.warn(msg, DeprecationWarning)
        self.name = None
        self.required = required or primary_key
        self.default = default
        self.unique = bool(unique or unique_with)
        self.unique_with = unique_with
        self.primary_key = primary_key
        self.validation = validation
        self.create_default = create_default
        self.choices = choices

    def __get__(self, instance, owner):
        """Descriptor for retrieving a value from a field in a document. Do 
        any necessary conversion between Python and MongoDB types.
        """
        if instance is None:
            # Document class being used rather than a document object
            return self

        # Get value from document instance if available, if not use default
        value = instance._data.get(self.name)
        if value is None:
            value = self.default
            # Allow callable default values
            if callable(value):
                value = value()
            
            # Check whether we should auto-create a default document
            if self.create_default and hasattr(self, 'document'):
                value = self.document()
        return value

    def __set__(self, instance, value):
        """Descriptor for assigning a value to a field in a document.
        """
        instance._data[self.name] = value

    def to_python(self, value):
        """Convert a MongoDB-compatible type to a Python type.
        """
        return value
<commit_msg>Removed create_default since it can be achieved with the `default` argument (like default=MyEmbeddedDocument since default takes callables too).<commit_after>from queryset import QuerySet, QuerySetManager
from queryset import DoesNotExist, MultipleObjectsReturned

import sys
import pymongo


_document_registry = {}

def get_document(name):
    return _document_registry[name]


class ValidationError(Exception):
    pass


class BaseField(object):
    """A base class for fields in a MongoDB document. Instances of this class
    may be added to subclasses of `Document` to define a document's schema.
    """

    # Fields may have _types inserted into indexes by default 
    _index_with_types = True
    
    def __init__(self, db_field=None, name=None, required=False, default=None, 
                 unique=False, unique_with=None, primary_key=False, validation=None,
                 choices=None):
        self.db_field = (db_field or name) if not primary_key else '_id'
        if name:
            import warnings
            msg = "Fields' 'name' attribute deprecated in favour of 'db_field'"
            warnings.warn(msg, DeprecationWarning)
        self.name = None
        self.required = required or primary_key
        self.default = default
        self.unique = bool(unique or unique_with)
        self.unique_with = unique_with
        self.primary_key = primary_key
        self.validation = validation
        self.choices = choices

    def __get__(self, instance, owner):
        """Descriptor for retrieving a value from a field in a document. Do 
        any necessary conversion between Python and MongoDB types.
        """
        if instance is None:
            # Document class being used rather than a document object
            return self

        # Get value from document instance if available, if not use default
        value = instance._data.get(self.name)
        if value is None:
            value = self.default
            # Allow callable default values
            if callable(value):
                value = value()
        return value

    def __set__(self, instance, value):
        """Descriptor for assigning a value to a field in a document.
        """
        instance._data[self.name] = value

    def to_python(self, value):
        """Convert a MongoDB-compatible type to a Python type.
        """
        return value

{'fraction_non_alphanumeric': 0.06692591076545232, 'fraction_numerical': 0.0, 'mean_word_length': 2.410327983251919, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4511007', 'n_tokens_mistral': 1256, 'n_tokens_neox': 1210, 'n_words': 461}