<commit_before>import logging
from contentcuration.models import *
from rest_framework import serializers
from rest_framework_bulk import BulkListSerializer, BulkSerializerMixin
from contentcuration.api import get_total_size, get_node_siblings, get_node_ancestors, get_child_names, count_files, count_all_children
from rest_framework.utils import model_meta
from collections import OrderedDict
from rest_framework.fields import set_value, SkipField
from rest_framework.exceptions import ValidationError
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db import transaction
from django.conf import settings

class LicenseSerializer(serializers.ModelSerializer):
    class Meta:
        model = License
        fields = ('license_name', 'exists', 'id')

class ChannelSerializer(serializers.ModelSerializer):
    resource_count = serializers.SerializerMethodField('count_resources')
    resource_size = serializers.SerializerMethodField('calculate_resources_size')

    def count_resources(self, channel):
        if not channel.draft:
            return 0
        else:
            return count_files(channel.draft.root_node)

    def calculate_resources_size(self, channel):
        if not channel.draft:
            return 0
        else:
            return get_total_size(channel.draft.root_node)

    class Meta:
        model = Channel
        fields = ('channel_id', 'name', 'description', 'editors', 'draft', 'clipboard', 'deleted', 'published','resource_count', 'resource_size', 'version', 'thumbnail')

class TopicTreeSerializer(serializers.ModelSerializer):
    name = serializers.SerializerMethodField('get_channel_name')

    def get_channel_name(self, tree):
        return tree.channel.name

    class Meta:
        model = TopicTree
        fields = ('name', 'channel', 'root_node', 'id')

class FileSerializer(serializers.ModelSerializer):
    content_copy = serializers.SerializerMethodField('get_file_url')

    def get(*args, **kwargs):
         return super.get(*args, **kwargs)
    class Meta:
        model = File
        fields = ('id', 'checksum', 'file_size', 'content_copy', 'contentnode', 'file_format', 'preset', 'lang','original_filename')

    def get_file_url(self, obj):
        return obj.content_copy.url

class FileFormatSerializer(serializers.ModelSerializer):
    class Meta:
        model = FileFormat
        fields = ("__all__")

class FormatPresetSerializer(serializers.ModelSerializer):
   # files = FileSerializer(many=True, read_only=True)

   class Meta:
        model = FormatPreset
        fields = ('id', 'readable_name', 'multi_language', 'supplementary', 'order', 'kind', 'allowed_formats')

class ContentKindSerializer(serializers.ModelSerializer):
   class Meta:
        model = ContentKind
        fields = ("__all__")

class CustomListSerializer(serializers.ListSerializer):
    def update(self, instance, validated_data):
        node_mapping = {node.id: node for node in instance}
        update_nodes = {}
        ret = []
        tag_names = []

        with transaction.atomic():
            for item in validated_data:
                tag_names += item.pop('tags')
                if 'id' in item:
                    update_nodes[item['id']] = item
                else:
                    # create new nodes
                    ret.append(ContentNode.objects.create(**item))

        # get all tags, if doesn't exist, create them.
        # this step is also needed for adding new tags to existing node.
        # in this case, we don't need the list of all_tags_pk, but we need to create the new tags.
        new_tags = []
        existing_tags = []
        tag_names = list(set(tag_names)) #get rid of repetitive tag_names
        for name in tag_names:
            tag_tuple = ContentTag.objects.get_or_create(tag_name=name)
            if tag_tuple[1]:
                new_tags.append(tag_tuple[0])
            else:
                existing_tags.append(tag_tuple[0])

        if ret:
            # new nodes and tags have been created, now add tags to them
            bulk_adding_list = []
            ThroughModel = ContentNode.tags.through
            all_tags = existing_tags + new_tags
            for tag in all_tags:
                for node in ret:
                    bulk_adding_list.append(ThroughModel(node_id=node.pk, contenttag_id=tag.pk))
            ThroughModel.objects.bulk_create(bulk_adding_list)

        # Perform updates.
        if update_nodes:
            for node_id, data in update_nodes.items():
                node = node_mapping.get(node_id, None)
                if node:
                    # potential optimization opportunity<commit_msg>modify CustomListSerializer to adapt the new schema on ContentTag<commit_after>import logging
from contentcuration.models import *
from rest_framework import serializers
from rest_framework_bulk import BulkListSerializer, BulkSerializerMixin
from contentcuration.api import get_total_size, get_node_siblings, get_node_ancestors, get_child_names, count_files, count_all_children
from rest_framework.utils import model_meta
from collections import OrderedDict
from rest_framework.fields import set_value, SkipField
from rest_framework.exceptions import ValidationError
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db import transaction
from django.conf import settings

class LicenseSerializer(serializers.ModelSerializer):
    class Meta:
        model = License
        fields = ('license_name', 'exists', 'id')

class ChannelSerializer(serializers.ModelSerializer):
    resource_count = serializers.SerializerMethodField('count_resources')
    resource_size = serializers.SerializerMethodField('calculate_resources_size')

    def count_resources(self, channel):
        if not channel.draft:
            return 0
        else:
            return count_files(channel.draft.root_node)

    def calculate_resources_size(self, channel):
        if not channel.draft:
            return 0
        else:
            return get_total_size(channel.draft.root_node)

    class Meta:
        model = Channel
        fields = ('channel_id', 'name', 'description', 'editors', 'draft', 'clipboard', 'deleted', 'published','resource_count', 'resource_size', 'version', 'thumbnail')

class TopicTreeSerializer(serializers.ModelSerializer):
    name = serializers.SerializerMethodField('get_channel_name')

    def get_channel_name(self, tree):
        return tree.channel.name

    class Meta:
        model = TopicTree
        fields = ('name', 'channel', 'root_node', 'id')

class FileSerializer(serializers.ModelSerializer):
    content_copy = serializers.SerializerMethodField('get_file_url')

    def get(*args, **kwargs):
         return super.get(*args, **kwargs)
    class Meta:
        model = File
        fields = ('id', 'checksum', 'file_size', 'content_copy', 'contentnode', 'file_format', 'preset', 'lang','original_filename')

    def get_file_url(self, obj):
        return obj.content_copy.url

class FileFormatSerializer(serializers.ModelSerializer):
    class Meta:
        model = FileFormat
        fields = ("__all__")

class FormatPresetSerializer(serializers.ModelSerializer):
   # files = FileSerializer(many=True, read_only=True)

   class Meta:
        model = FormatPreset
        fields = ('id', 'readable_name', 'multi_language', 'supplementary', 'order', 'kind', 'allowed_formats')

class ContentKindSerializer(serializers.ModelSerializer):
   class Meta:
        model = ContentKind
        fields = ("__all__")

class CustomListSerializer(serializers.ListSerializer):
    def update(self, instance, validated_data):
        node_mapping = {node.id: node for node in instance}
        update_nodes = {}
        ret = []
        tag_names = []

        with transaction.atomic():
            for item in validated_data:
                tag_names += item.pop('tags')
                if 'id' in item:
                    update_nodes[item['id']] = item
                else:
                    # create new nodes
                    ret.append(ContentNode.objects.create(**item))

        # get all tags, if doesn't exist, create them.
        # this step is also needed for adding new tags to existing node.
        # in this case, we don't need the list of all_tags_pk, but we need to create the new tags.
        new_tags = []
        existing_tags = []
        tag_names = list(set(tag_names)) #get rid of repetitive tag_names
        for name in tag_names:
            tag_tuple = ContentTag.objects.get_or_create(tag_name=name, channel=instance[0].get_root().channel_main.all()[0].id)
            if tag_tuple[1]:
                new_tags.append(tag_tuple[0])
            else:
                existing_tags.append(tag_tuple[0])

        if ret:
            # new nodes and tags have been created, now add tags to them
            bulk_adding_list = []
            ThroughModel = ContentNode.tags.through
            all_tags = existing_tags + new_tags
            for tag in all_tags:
                for node in ret:
                    bulk_adding_list.append(ThroughModel(node_id=node.pk, contenttag_id=tag.pk))
            ThroughModel.objects.bulk_create(bulk_adding_list)

        # Perform updates.
        if update_nodes:
            for node_id, data in update_nodes.items():
                node = node_mapping.get(node_id, None)
                if node:
                    # potential optimization opportunity
{'fraction_non_alphanumeric': 0.0878656669130848, 'fraction_numerical': 0.001267293272784877, 'mean_word_length': 2.8684640522875817, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3305197', 'n_tokens_mistral': 2567, 'n_tokens_neox': 2492, 'n_words': 664}