<commit_before>import json
import re
from collections import OrderedDict

from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.core.exceptions import ValidationError as DjangoValidationError
from django.core.files.storage import default_storage
from django.db import transaction
from django.db.models import Max
from django.db.models import Sum
from le_utils.constants import content_kinds
from le_utils.constants import exercises
from le_utils.constants import roles
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
from rest_framework.fields import set_value
from rest_framework.fields import SkipField
from rest_framework.settings import api_settings
from rest_framework.utils import model_meta
from rest_framework_bulk import BulkSerializerMixin

from contentcuration.models import AssessmentItem
from contentcuration.models import Channel
from contentcuration.models import ChannelSet
from contentcuration.models import ContentKind
from contentcuration.models import ContentNode
from contentcuration.models import ContentTag
from contentcuration.models import File
from contentcuration.models import FileFormat
from contentcuration.models import FormatPreset
from contentcuration.models import generate_object_storage_name
from contentcuration.models import generate_storage_url
from contentcuration.models import Invitation
from contentcuration.models import Language
from contentcuration.models import License
from contentcuration.models import PrerequisiteContentRelationship
from contentcuration.models import SecretToken
from contentcuration.models import User
from contentcuration.statistics import record_node_addition_stats
from contentcuration.utils.channelcache import ChannelCacher
from contentcuration.utils.format import format_size


class LicenseSerializer(serializers.ModelSerializer):

    class Meta:
        model = License
        fields = ('license_name', 'exists', 'id', 'license_url', 'license_description', 'copyright_holder_required', 'is_custom')


class LanguageSerializer(serializers.ModelSerializer):
    id = serializers.CharField(required=False)
    ietf_name = serializers.SerializerMethodField('generate_ietf_name')

    def generate_ietf_name(self, language):
        return str(language)

    class Meta:
        model = Language
        fields = ('lang_code', 'lang_subcode', 'id', 'readable_name', 'ietf_name', 'native_name')


class FileFormatSerializer(serializers.ModelSerializer):

    class Meta:
        model = FileFormat
        fields = ("__all__")


class FormatPresetSerializer(serializers.ModelSerializer):
    # files = FileSerializer(many=True, read_only=True)
    associated_mimetypes = serializers.SerializerMethodField('retrieve_mimetypes')
    # Handles multi-language content (Backbone won't allow duplicate ids in collection, so name retains id)
    name = serializers.SerializerMethodField('retrieve_name')

    def retrieve_mimetypes(self, preset):
        return preset.allowed_formats.values_list('mimetype', flat=True)

    def retrieve_name(self, preset):
        return preset.id

    class Meta:
        model = FormatPreset
        fields = (
            'id', 'name', 'readable_name', 'multi_language', 'supplementary', 'thumbnail', 'subtitle', 'order', 'kind',
            'allowed_formats', 'associated_mimetypes', 'display')


class FileListSerializer(serializers.ListSerializer):

    def update(self, instance, validated_data):
        ret = []
        with transaction.atomic():
            # Get files that have the same contentnode, preset, and language as the files that are now attached to this node
            for item in validated_data:
                file_obj = File.objects.get(pk=item['id'])
                file_obj.language_id = item.get('language') and item['language']['id']
                file_obj.contentnode = item['contentnode']

                # Make sure file exists
                file_path = generate_object_storage_name(file_obj.checksum, str(file_obj))
                if not default_storage.exists(file_path):
                    raise OSError("Error: file {} was not found".format(str(file_obj)))

                files_to_replace = item['contentnode'].files.exclude(pk=file_obj.pk)\
                    .filter(preset_id=file_obj.preset_id, language_id=file_obj.language_id)
                files_to_replace.delete()

                file_obj.save()
                ret.append(file_obj)

        return ret


class FileSerializer(BulkSerializerMixin, serializers.ModelSerializer):
    file_on_disk = serializers.SerializerMethodField('get_file_url')
    storage_url = serializers.SerializerMethodField('retrieve_storage_url')
    mimetype = serializers.SerializerMethodField('retrieve_extension')
    language = LanguageSerializer(many=False, required=False, allow_null=True)
    display_name = serializers.SerializerMethodField('retrieve_display_name')
    id = serializers.CharField(required=False)
    preset = FormatPresetSerializer(many=False, read_only=True)

    def get(*args, **kwargs):
        return super.get(*args, **kwargs)

    def get_file_url(self, obj):
        return obj.file_on_disk.url<commit_msg>Fixed removing files<commit_after>import json
import re
from collections import OrderedDict

from django.conf import settings
from django.core.exceptions import ObjectDoesNotExist
from django.core.exceptions import ValidationError as DjangoValidationError
from django.core.files.storage import default_storage
from django.db import transaction
from django.db.models import Max
from django.db.models import Sum
from le_utils.constants import content_kinds
from le_utils.constants import exercises
from le_utils.constants import roles
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
from rest_framework.fields import set_value
from rest_framework.fields import SkipField
from rest_framework.settings import api_settings
from rest_framework.utils import model_meta
from rest_framework_bulk import BulkSerializerMixin

from contentcuration.models import AssessmentItem
from contentcuration.models import Channel
from contentcuration.models import ChannelSet
from contentcuration.models import ContentKind
from contentcuration.models import ContentNode
from contentcuration.models import ContentTag
from contentcuration.models import File
from contentcuration.models import FileFormat
from contentcuration.models import FormatPreset
from contentcuration.models import generate_object_storage_name
from contentcuration.models import generate_storage_url
from contentcuration.models import Invitation
from contentcuration.models import Language
from contentcuration.models import License
from contentcuration.models import PrerequisiteContentRelationship
from contentcuration.models import SecretToken
from contentcuration.models import User
from contentcuration.statistics import record_node_addition_stats
from contentcuration.utils.channelcache import ChannelCacher
from contentcuration.utils.format import format_size


class LicenseSerializer(serializers.ModelSerializer):

    class Meta:
        model = License
        fields = ('license_name', 'exists', 'id', 'license_url', 'license_description', 'copyright_holder_required', 'is_custom')


class LanguageSerializer(serializers.ModelSerializer):
    id = serializers.CharField(required=False)
    ietf_name = serializers.SerializerMethodField('generate_ietf_name')

    def generate_ietf_name(self, language):
        return str(language)

    class Meta:
        model = Language
        fields = ('lang_code', 'lang_subcode', 'id', 'readable_name', 'ietf_name', 'native_name')


class FileFormatSerializer(serializers.ModelSerializer):

    class Meta:
        model = FileFormat
        fields = ("__all__")


class FormatPresetSerializer(serializers.ModelSerializer):
    # files = FileSerializer(many=True, read_only=True)
    associated_mimetypes = serializers.SerializerMethodField('retrieve_mimetypes')
    # Handles multi-language content (Backbone won't allow duplicate ids in collection, so name retains id)
    name = serializers.SerializerMethodField('retrieve_name')

    def retrieve_mimetypes(self, preset):
        return preset.allowed_formats.values_list('mimetype', flat=True)

    def retrieve_name(self, preset):
        return preset.id

    class Meta:
        model = FormatPreset
        fields = (
            'id', 'name', 'readable_name', 'multi_language', 'supplementary', 'thumbnail', 'subtitle', 'order', 'kind',
            'allowed_formats', 'associated_mimetypes', 'display')


class FileListSerializer(serializers.ListSerializer):

    def update(self, instance, validated_data):
        ret = []
        nodes_to_check = []
        with transaction.atomic():
            # Get files that have the same contentnode, preset, and language as the files that are now attached to this node
            for item in validated_data:
                file_obj = File.objects.get(pk=item['id'])
                file_obj.language_id = item.get('language') and item['language']['id']
                file_obj.contentnode = item['contentnode']

                if item['contentnode'] not in nodes_to_check:
                    nodes_to_check.append(item['contentnode'])

                # Make sure file exists
                file_path = generate_object_storage_name(file_obj.checksum, str(file_obj))
                if not default_storage.exists(file_path):
                    raise OSError("Error: file {} was not found".format(str(file_obj)))

                # Replace existing files
                files_to_replace = item['contentnode'].files.exclude(pk=file_obj.pk)\
                    .filter(preset_id=file_obj.preset_id, language_id=file_obj.language_id)
                files_to_replace.delete()

                file_obj.save()
                ret.append(file_obj)

            # Remove items that are not in the validated data (file has been removed)
            for node in nodes_to_check:
                file_ids = [f['id'] for f in validated_data if f['contentnode'].pk == node.pk]
                node.files.exclude(pk__in=file_ids).delete()

        return ret


class FileSerializer(BulkSerializerMixin, serializers.ModelSerializer):
    file_on_disk = serializers.SerializerMethodField('get_file_url')
    storage_url = serializers.SerializerMethodField('retrieve_storage_url')
    mimetype = serializers.SerializerMethodField('retrieve_extension')
    language = LanguageSerializer(many=False, required=False, allow_null=True)
    display_name = serializers.SerializerMethodField('retrieve_display_name')
    id = serializers.CharField(required=False)
    preset = FormatPresetSerializer(many=False, read_only=True)

    def get(*args, **kwargs):
        return super.get(*args, **kwargs)

    def get_file_url(self, obj):
        return obj.file_on_disk.url
{'fraction_non_alphanumeric': 0.07480999908433293, 'fraction_numerical': 0.0, 'mean_word_length': 4.21585482330468, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '405733', 'n_tokens_mistral': 2906, 'n_tokens_neox': 2808, 'n_words': 738}