<commit_before>import os
import itertools
from collections import Callable
from functools import reduce

from django.utils.datastructures import SortedDict
from django.forms.forms import BaseForm, get_declared_fields, NON_FIELD_ERRORS, pretty_name
from django.forms.widgets import media_property
from django.core.exceptions import FieldError
from django.core.validators import EMPTY_VALUES
from django.forms.util import ErrorList
from django.forms.formsets import BaseFormSet, formset_factory
from django.utils.translation import ugettext_lazy as _, ugettext
from django.utils.text import capfirst

from mongoengine.fields import ObjectIdField, ListField, ReferenceField, FileField, MapField
try:
    from mongoengine.base import ValidationError
except ImportError:
    from mongoengine.errors import ValidationError
from mongoengine.queryset import OperationError, Q
from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME
from gridfs import GridFS

from .documentoptions import DocumentMetaWrapper
from .util import with_metaclass, load_field_generator

_fieldgenerator = load_field_generator()

def _get_unique_filename(name, db_alias=DEFAULT_CONNECTION_NAME, collection_name='fs'):
    fs = GridFS(get_db(db_alias), collection_name)
    file_root, file_ext = os.path.splitext(name)
    count = itertools.count(1)
    while fs.exists(filename=name):
        # file_ext includes the dot.
        name = os.path.join("%s_%s%s" % (file_root, next(count), file_ext))
    return name

# The awesome Mongoengine ImageGridFsProxy wants to pull a field
# from a document to get necessary data. Trouble is that this doesn't work
# if the ImageField is stored on List or MapField. So we pass a nice fake
# document to the proxy to get saving the file done. Yeah it really is that ugly.
class FakeDocument(object):
    _fields = {}
    
    def __init__(self, key, field):
        super(FakeDocument, self).__init__()
        
        self._fields[key] = field
    
    # We don't care if anything gets marked on this
    # we do update a real field later though. That should
    # trigger the same thing on the real document.
    def _mark_as_changed(self, key):
        pass
    
def _save_iterator_file(field, uploaded_file, file_data=None):
    """
    Takes care of saving a file for a list field. Returns a Mongoengine
    fileproxy object or the file field.
    """
    fake_document = FakeDocument(field.name, field.field)
    overwrote_instance = False
    overwrote_key = False
    # for a new file we need a new proxy object
    if file_data is None:
        file_data = field.field.proxy_class(db_alias=field.field.db_alias,
                            collection_name=field.field.collection_name)
    
    # overwrite an existing file
    if file_data.instance is None:
        file_data.instance = fake_document
        overwrote_instance = True
    if file_data.key is None:
        file_data.key = field.name
        overwrote_key = True
    
    if file_data.grid_id:
        file_data.delete()
        
    uploaded_file.seek(0)
    filename = _get_unique_filename(uploaded_file.name, field.field.db_alias, field.field.collection_name)
    file_data.put(uploaded_file, content_type=uploaded_file.content_type, filename=filename)
    file_data.close()
    
    if overwrote_instance:
        file_data.instance = None
    if overwrote_key:
        file_data.key = None
        
    return file_data

def construct_instance(form, instance, fields=None, exclude=None, ignore=None):
    """
    Constructs and returns a document instance from the bound ``form``'s
    ``cleaned_data``, but does not save the returned instance to the
    database.
    """
    cleaned_data = form.cleaned_data
    file_field_list = []
    
    # check wether object is instantiated
    if isinstance(instance, type):
        instance = instance()
        
    for f in instance._fields.values():
        if isinstance(f, ObjectIdField):
            continue
        if not f.name in cleaned_data:
            continue
        if fields is not None and f.name not in fields:
            continue
        if exclude and f.name in exclude:
            continue
        # Defer saving file-type fields until after the other fields, so a
        # callable upload_to can use the values from other fields.
        if isinstance(f, FileField) or (isinstance(f, (MapField, ListField)) and isinstance(f.field, FileField)):
            file_field_list.append(f)
        else:
            setattr(instance, f.name, cleaned_data.get(f.name))

    for f in file_field_list:
        if isinstance(f, MapField):
            map_field = getattr(instance, f.name)
            uploads = cleaned_data[f.name]
            for key, uploaded_file in uploads.items():
                if uploaded_file is None:
                    continue
                file_data = map_field.get(key, None)
                map_field[key] = _save_iterator_file(f, uploaded_file, file_data)
            setattr(instance, f.name, map_field)
        elif isinstance(f, ListField):
            list_field = getattr(instance, f.name)
            uploads = cleaned_data[f.name]
            for i, uploaded_file in enumerate(uploads):
                if uploaded_file is None:
                    continue
                try:
                    file_data = list_field[i]
                except IndexError:
                    file_data = None
                file_obj = _save_iterator_file(f, uploaded_file, file_data)
                try:
                    list_field[i] = file_obj
                except IndexError:
                    list_field.append(file_obj)
            setattr(instance, f.name, list_field)
        else:
            field = getattr(instance, f.name)
            upload = cleaned_data[f.name]
            if upload is None:
                continue
            
            try:
                upload.file.seek(0)
                # delete first to get the names right
                if field.grid_id:
                    field.delete()
                filename = _get_unique_filename(upload.name, f.db_alias, f.collection_name)
                field.put(upload, content_type=upload.content_type, filename=filename)
                setattr(instance, f.name, field)
            except AttributeError:
                # file was already uploaded and not changed during edit.
                # upload is already the gridfsproxy object we need.
                upload.get()
                setattr(instance, f.name, upload)
            
    return instance


def save_instance(form, instance, fields=None, fail_message='saved',
                  commit=True, exclude=None, construct=True):
    """
    Saves bound Form ``form``'s cleaned_data into document instance ``instance``.

    If commit=True, then the changes to ``instance`` will be saved to the
    database. Returns ``instance``.

    If construct=False, assume ``instance`` has already been constructed and
    just needs to be saved.
    """
    instance = construct_instance(form, instance, fields, exclude)
    if form.errors:
        raise ValueError("The %s could not be %s because the data didn't"
                         " validate." % (instance.__class__.__name__, fail_message))
    
    if commit and hasattr(instance, 'save'):
        # see BaseDocumentForm._post_clean for an explanation
        if hasattr(form, '_delete_before_save'):
            data = instance._data
            new_data = dict([(n, f) for n, f in data.items() if not n in form._delete_before_save])
            if hasattr(instance, '_changed_fields'):
                for field in form._delete_before_save:
                    try:
                        instance._changed_fields.remove(field)
                    except ValueError:
                        pass
            instance._data = new_data
            instance.save()
            instance._data = data
        else:<commit_msg>Don't construct the instance several times during clean and save<commit_after>import os
import itertools
from collections import Callable
from functools import reduce

from django.utils.datastructures import SortedDict
from django.forms.forms import BaseForm, get_declared_fields, NON_FIELD_ERRORS, pretty_name
from django.forms.widgets import media_property
from django.core.exceptions import FieldError
from django.core.validators import EMPTY_VALUES
from django.forms.util import ErrorList
from django.forms.formsets import BaseFormSet, formset_factory
from django.utils.translation import ugettext_lazy as _, ugettext
from django.utils.text import capfirst

from mongoengine.fields import ObjectIdField, ListField, ReferenceField, FileField, MapField
try:
    from mongoengine.base import ValidationError
except ImportError:
    from mongoengine.errors import ValidationError
from mongoengine.queryset import OperationError, Q
from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME
from gridfs import GridFS

from .documentoptions import DocumentMetaWrapper
from .util import with_metaclass, load_field_generator

_fieldgenerator = load_field_generator()

def _get_unique_filename(name, db_alias=DEFAULT_CONNECTION_NAME, collection_name='fs'):
    fs = GridFS(get_db(db_alias), collection_name)
    file_root, file_ext = os.path.splitext(name)
    count = itertools.count(1)
    while fs.exists(filename=name):
        # file_ext includes the dot.
        name = os.path.join("%s_%s%s" % (file_root, next(count), file_ext))
    return name

# The awesome Mongoengine ImageGridFsProxy wants to pull a field
# from a document to get necessary data. Trouble is that this doesn't work
# if the ImageField is stored on List or MapField. So we pass a nice fake
# document to the proxy to get saving the file done. Yeah it really is that ugly.
class FakeDocument(object):
    _fields = {}
    
    def __init__(self, key, field):
        super(FakeDocument, self).__init__()
        
        self._fields[key] = field
    
    # We don't care if anything gets marked on this
    # we do update a real field later though. That should
    # trigger the same thing on the real document.
    def _mark_as_changed(self, key):
        pass
    
def _save_iterator_file(field, uploaded_file, file_data=None):
    """
    Takes care of saving a file for a list field. Returns a Mongoengine
    fileproxy object or the file field.
    """
    fake_document = FakeDocument(field.name, field.field)
    overwrote_instance = False
    overwrote_key = False
    # for a new file we need a new proxy object
    if file_data is None:
        file_data = field.field.proxy_class(db_alias=field.field.db_alias,
                            collection_name=field.field.collection_name)
    
    if file_data.instance is None:
        file_data.instance = fake_document
        overwrote_instance = True
    if file_data.key is None:
        file_data.key = field.name
        overwrote_key = True
    
    if file_data.grid_id:
        file_data.delete()
        
    uploaded_file.seek(0)
    filename = _get_unique_filename(uploaded_file.name, field.field.db_alias, field.field.collection_name)
    file_data.put(uploaded_file, content_type=uploaded_file.content_type, filename=filename)
    file_data.close()
    
    if overwrote_instance:
        file_data.instance = None
    if overwrote_key:
        file_data.key = None
        
    return file_data

def construct_instance(form, instance, fields=None, exclude=None, ignore=None):
    """
    Constructs and returns a document instance from the bound ``form``'s
    ``cleaned_data``, but does not save the returned instance to the
    database.
    """
    cleaned_data = form.cleaned_data
    file_field_list = []
    
    # check wether object is instantiated
    if isinstance(instance, type):
        instance = instance()
        
    for f in instance._fields.values():
        if isinstance(f, ObjectIdField):
            continue
        if not f.name in cleaned_data:
            continue
        if fields is not None and f.name not in fields:
            continue
        if exclude and f.name in exclude:
            continue
        # Defer saving file-type fields until after the other fields, so a
        # callable upload_to can use the values from other fields.
        if isinstance(f, FileField) or (isinstance(f, (MapField, ListField)) and isinstance(f.field, FileField)):
            file_field_list.append(f)
        else:
            setattr(instance, f.name, cleaned_data.get(f.name))

    for f in file_field_list:
        if isinstance(f, MapField):
            map_field = getattr(instance, f.name)
            uploads = cleaned_data[f.name]
            for key, uploaded_file in uploads.items():
                if uploaded_file is None:
                    continue
                file_data = map_field.get(key, None)
                map_field[key] = _save_iterator_file(f, uploaded_file, file_data)
            setattr(instance, f.name, map_field)
        elif isinstance(f, ListField):
            list_field = getattr(instance, f.name)
            uploads = cleaned_data[f.name]
            for i, uploaded_file in enumerate(uploads):
                if uploaded_file is None:
                    continue
                try:
                    file_data = list_field[i]
                except IndexError:
                    file_data = None
                file_obj = _save_iterator_file(f, uploaded_file, file_data)
                try:
                    list_field[i] = file_obj
                except IndexError:
                    list_field.append(file_obj)
            setattr(instance, f.name, list_field)
        else:
            field = getattr(instance, f.name)
            upload = cleaned_data[f.name]
            if upload is None:
                continue
            
            try:
                upload.file.seek(0)
                # delete first to get the names right
                if field.grid_id:
                    field.delete()
                filename = _get_unique_filename(upload.name, f.db_alias, f.collection_name)
                field.put(upload, content_type=upload.content_type, filename=filename)
                setattr(instance, f.name, field)
            except AttributeError:
                # file was already uploaded and not changed during edit.
                # upload is already the gridfsproxy object we need.
                upload.get()
                setattr(instance, f.name, upload)
            
    return instance


def save_instance(form, instance, fields=None, fail_message='saved',
                  commit=True, exclude=None, construct=True):
    """
    Saves bound Form ``form``'s cleaned_data into document instance ``instance``.

    If commit=True, then the changes to ``instance`` will be saved to the
    database. Returns ``instance``.

    If construct=False, assume ``instance`` has already been constructed and
    just needs to be saved.
    """
    if construct:
        instance = construct_instance(form, instance, fields, exclude)
        
    if form.errors:
        raise ValueError("The %s could not be %s because the data didn't"
                         " validate." % (instance.__class__.__name__, fail_message))
    
    if commit and hasattr(instance, 'save'):
        # see BaseDocumentForm._post_clean for an explanation
        if hasattr(form, '_delete_before_save'):
            data = instance._data
            new_data = dict([(n, f) for n, f in data.items() if not n in form._delete_before_save])
            if hasattr(instance, '_changed_fields'):
                for field in form._delete_before_save:
                    try:
                        instance._changed_fields.remove(field)
                    except ValueError:
                        pass
            instance._data = new_data
            instance.save()
            instance._data = data
        else:
{'fraction_non_alphanumeric': 0.07148705674875606, 'fraction_numerical': 0.0003779051458084021, 'mean_word_length': 2.366836301950806, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3679391', 'n_tokens_mistral': 4369, 'n_tokens_neox': 4238, 'n_words': 1316}