<commit_before>import logging
import json
import re
from contentcuration.models import *
from rest_framework import serializers
from rest_framework_bulk import BulkListSerializer, BulkSerializerMixin
from contentcuration.api import get_total_size, get_node_ancestors, count_files, calculate_node_metadata, clean_db, recurse
from rest_framework.utils import model_meta
from collections import OrderedDict
from rest_framework.fields import set_value, SkipField
from rest_framework.exceptions import ValidationError
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db import transaction
from django.conf import settings
from django.core.files import File as DjFile

class LicenseSerializer(serializers.ModelSerializer):
    class Meta:
        model = License
        fields = ('license_name', 'exists', 'id', 'license_url', 'license_description')

class LanguageSerializer(serializers.ModelSerializer):
    id = serializers.CharField(required=False)

    class Meta:
        model = Language
        fields = ('lang_code', 'lang_subcode', 'id', 'readable_name')

class FileFormatSerializer(serializers.ModelSerializer):
    class Meta:
        model = FileFormat
        fields = ("__all__")

class FormatPresetSerializer(serializers.ModelSerializer):
   # files = FileSerializer(many=True, read_only=True)
    associated_mimetypes = serializers.SerializerMethodField('retrieve_mimetypes')
    # Handles multi-language content (Backbone won't allow duplicate ids in collection, so name retains id)
    name = serializers.SerializerMethodField('retrieve_name')

    def retrieve_mimetypes(self, preset):
        mimetypes = []
        for m in preset.allowed_formats.all():
            mimetypes.append(m.mimetype)
        return mimetypes

    def retrieve_name(self, preset):
        return preset.id

    class Meta:
        model = FormatPreset
        fields = ('id', 'name', 'readable_name', 'multi_language', 'supplementary', 'subtitle', 'order', 'kind', 'allowed_formats','associated_mimetypes', 'display')

class FileListSerializer(serializers.ListSerializer):
    def update(self, instance, validated_data):
        ret = []
        update_files = {}

        with transaction.atomic():
            for item in validated_data:
                item.update({
                    'preset_id' : item['preset']['id'],
                    'language_id' : item.get('language')['id'] if item.get('language') else None
                })
                del item['preset']
                del item['language']

                if 'id' in item:
                    update_files[item['id']] = item
                else:
                    # create new nodes
                    ret.append(File.objects.create(**item))

        for file_obj in validated_data:
            contentnode = file_obj['contentnode']
            preset = file_obj['preset_id']
            file_id = file_obj['id']
            language = file_obj.get('language_id')
            files_to_delete = File.objects.filter(Q(contentnode=contentnode) & (Q(preset_id=preset) | Q(preset=None)) & (Q(language_id=language)) & ~Q(id=file_id))
            for to_delete in files_to_delete:
                to_delete.delete()

        if update_files:
            with transaction.atomic():
                for file_id, data in update_files.items():
                    file_obj, is_new = File.objects.get_or_create(pk=file_id)
                    # potential optimization opportunity
                    for attr, value in data.items():
                        setattr(file_obj, attr, value)
                    file_path = generate_file_on_disk_name(file_obj.checksum, str(file_obj))
                    if os.path.isfile(file_path):
                        file_obj.file_on_disk = DjFile(open(file_path, 'rb'))<commit_msg>Removed call to delete node language/preset<commit_after>import logging
import json
import re
from contentcuration.models import *
from rest_framework import serializers
from rest_framework_bulk import BulkListSerializer, BulkSerializerMixin
from contentcuration.api import get_total_size, get_node_ancestors, count_files, calculate_node_metadata, clean_db, recurse
from rest_framework.utils import model_meta
from collections import OrderedDict
from rest_framework.fields import set_value, SkipField
from rest_framework.exceptions import ValidationError
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db import transaction
from django.conf import settings
from django.core.files import File as DjFile

class LicenseSerializer(serializers.ModelSerializer):
    class Meta:
        model = License
        fields = ('license_name', 'exists', 'id', 'license_url', 'license_description')

class LanguageSerializer(serializers.ModelSerializer):
    id = serializers.CharField(required=False)

    class Meta:
        model = Language
        fields = ('lang_code', 'lang_subcode', 'id', 'readable_name')

class FileFormatSerializer(serializers.ModelSerializer):
    class Meta:
        model = FileFormat
        fields = ("__all__")

class FormatPresetSerializer(serializers.ModelSerializer):
   # files = FileSerializer(many=True, read_only=True)
    associated_mimetypes = serializers.SerializerMethodField('retrieve_mimetypes')
    # Handles multi-language content (Backbone won't allow duplicate ids in collection, so name retains id)
    name = serializers.SerializerMethodField('retrieve_name')

    def retrieve_mimetypes(self, preset):
        mimetypes = []
        for m in preset.allowed_formats.all():
            mimetypes.append(m.mimetype)
        return mimetypes

    def retrieve_name(self, preset):
        return preset.id

    class Meta:
        model = FormatPreset
        fields = ('id', 'name', 'readable_name', 'multi_language', 'supplementary', 'subtitle', 'order', 'kind', 'allowed_formats','associated_mimetypes', 'display')

class FileListSerializer(serializers.ListSerializer):
    def update(self, instance, validated_data):
        ret = []
        update_files = {}

        with transaction.atomic():
            for item in validated_data:
                item.update({
                    'preset_id' : item['preset']['id'],
                    'language_id' : item.get('language')['id'] if item.get('language') else None
                })

                if 'id' in item:
                    update_files[item['id']] = item
                else:
                    # create new nodes
                    ret.append(File.objects.create(**item))

        for file_obj in validated_data:
            contentnode = file_obj['contentnode']
            preset = file_obj['preset_id']
            file_id = file_obj['id']
            language = file_obj.get('language_id')
            files_to_delete = File.objects.filter(Q(contentnode=contentnode) & (Q(preset_id=preset) | Q(preset=None)) & (Q(language_id=language)) & ~Q(id=file_id))
            for to_delete in files_to_delete:
                to_delete.delete()

        if update_files:
            with transaction.atomic():
                for file_id, data in update_files.items():
                    file_obj, is_new = File.objects.get_or_create(pk=file_id)
                    # potential optimization opportunity
                    for attr, value in data.items():
                        setattr(file_obj, attr, value)
                    file_path = generate_file_on_disk_name(file_obj.checksum, str(file_obj))
                    if os.path.isfile(file_path):
                        file_obj.file_on_disk = DjFile(open(file_path, 'rb'))
{'fraction_non_alphanumeric': 0.09226112134772309, 'fraction_numerical': 0.0, 'mean_word_length': 2.832072617246596, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5620306', 'n_tokens_mistral': 2095, 'n_tokens_neox': 2020, 'n_words': 486}