<commit_before>import logging
import json
import re
from contentcuration.models import *
from rest_framework import serializers
from rest_framework_bulk import BulkListSerializer, BulkSerializerMixin
from contentcuration.api import get_total_size, get_node_ancestors, count_files, calculate_node_metadata
from rest_framework.utils import model_meta
from collections import OrderedDict
from rest_framework.fields import set_value, SkipField
from rest_framework.exceptions import ValidationError
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db import transaction
from django.conf import settings

class LicenseSerializer(serializers.ModelSerializer):
    class Meta:
        model = License
        fields = ('license_name', 'exists', 'id', 'license_url', 'license_description')

class LanguageSerializer(serializers.ModelSerializer):
    class Meta:
        model = Language
        fields = ('lang_code', 'lang_subcode', 'id')

class UserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = ('email', 'first_name', 'last_name', 'is_active', 'is_admin', 'id','clipboard_tree')

class ChannelSerializer(serializers.ModelSerializer):
    resource_count = serializers.SerializerMethodField('count_resources')
    resource_size = serializers.SerializerMethodField('calculate_resources_size')

    def count_resources(self, channel):
        if not channel.main_tree:
            return 0
        else:
            return count_files(channel.main_tree)

    def calculate_resources_size(self, channel):
        if not channel.main_tree:
            return 0
        else:
            return get_total_size(channel.main_tree)

    class Meta:
        model = Channel
        fields = ('id', 'name', 'description', 'editors', 'main_tree',
                    'clipboard_tree', 'trash_tree','resource_count', 'resource_size',
                    'version', 'thumbnail', 'deleted', 'public', 'pending_editors')

class FileSerializer(serializers.ModelSerializer):
    file_on_disk = serializers.SerializerMethodField('get_file_url')
    recommended_kind = serializers.SerializerMethodField('retrieve_recommended_kind')
    mimetype = serializers.SerializerMethodField('retrieve_extension')

    def get(*args, **kwargs):
         return super.get(*args, **kwargs)
    class Meta:
        model = File
        fields = ('id', 'checksum', 'file_size', 'file_on_disk', 'contentnode', 'file_format', 'preset', 'original_filename','recommended_kind', 'mimetype')

    def get_file_url(self, obj):
        return obj.file_on_disk.url

    def retrieve_recommended_kind(self, obj):
        return FormatPreset.objects.filter(allowed_formats__extension__contains=obj.file_format).first().kind.pk

    def retrieve_extension(self, obj):
        return obj.file_format.mimetype

class FileFormatSerializer(serializers.ModelSerializer):
    class Meta:
        model = FileFormat
        fields = ("__all__")

class FormatPresetSerializer(serializers.ModelSerializer):
   # files = FileSerializer(many=True, read_only=True)
    associated_mimetypes = serializers.SerializerMethodField('retrieve_mimetypes')

    def retrieve_mimetypes(self, preset):
        mimetypes = []
        for m in preset.allowed_formats.all():
            mimetypes.append(m.mimetype)
        return mimetypes

    class Meta:
        model = FormatPreset
        fields = ('id', 'readable_name', 'multi_language', 'supplementary', 'order', 'kind', 'allowed_formats','associated_mimetypes')

class ContentKindSerializer(serializers.ModelSerializer):
    associated_presets = serializers.SerializerMethodField('retrieve_associated_presets')
    def retrieve_associated_presets(self, kind):
        return FormatPreset.objects.filter(kind = kind).values()

    class Meta:
        model = ContentKind
        fields = ("kind", 'associated_presets')

class CustomListSerializer(serializers.ListSerializer):
    def update(self, instance, validated_data):
        node_mapping = {node.id: node for node in instance}
        update_nodes = {}
        tag_mapping = {}
        ret = []
        unformatted_input_tags = []

        with transaction.atomic():
            for item in validated_data:
                item_tags = item.get('tags')
                unformatted_input_tags += item.pop('tags')
                if 'id' in item:
                    update_nodes[item['id']] = item
                    tag_mapping[item['id']] = item_tags
                else:
                    # create new nodes
                    ret.append(ContentNode.objects.create(**item))

        # get all ContentTag objects, if doesn't exist, create them.
        all_tags = []

        for tag_data in unformatted_input_tags:
            # when deleting nodes, tag_data is a dict, but when adding nodes, it's a unicode string
            if isinstance(tag_data, unicode):
                print tag_data
                tag_data = json.loads(tag_data)
            tag_tuple = ContentTag.objects.get_or_create(tag_name=tag_data['tag_name'], channel_id=tag_data['channel'])
            all_tags.append(tag_tuple[0])

        if ret:
            # new nodes and tags have been created, now add tags to them
            bulk_adding_list = []
            ThroughModel = ContentNode.tags.through
            for tag in all_tags:
                for node in ret:
                    bulk_adding_list.append(ThroughModel(node_id=node.pk, contenttag_id=tag.pk))
            ThroughModel.objects.bulk_create(bulk_adding_list)

        # Perform updates.
        if update_nodes:
            for node_id, data in update_nodes.items():
                node = node_mapping.get(node_id, None)
                if node:
                    # potential optimization opportunity
                    for attr, value in data.items():
                        setattr(node, attr, value)
                    taglist = []
                    for tag_data in tag_mapping.get(node_id, None):
                        # when deleting nodes, tag_data is a dict, but when adding nodes, it's a unicode string
                        if isinstance(tag_data, unicode):
                            tag_data = json.loads(tag_data)<commit_msg>Removed print statement from serializer<commit_after>import logging
import json
import re
from contentcuration.models import *
from rest_framework import serializers
from rest_framework_bulk import BulkListSerializer, BulkSerializerMixin
from contentcuration.api import get_total_size, get_node_ancestors, count_files, calculate_node_metadata
from rest_framework.utils import model_meta
from collections import OrderedDict
from rest_framework.fields import set_value, SkipField
from rest_framework.exceptions import ValidationError
from django.core.exceptions import ValidationError as DjangoValidationError
from django.db import transaction
from django.conf import settings

class LicenseSerializer(serializers.ModelSerializer):
    class Meta:
        model = License
        fields = ('license_name', 'exists', 'id', 'license_url', 'license_description')

class LanguageSerializer(serializers.ModelSerializer):
    class Meta:
        model = Language
        fields = ('lang_code', 'lang_subcode', 'id')

class UserSerializer(serializers.ModelSerializer):
    class Meta:
        model = User
        fields = ('email', 'first_name', 'last_name', 'is_active', 'is_admin', 'id','clipboard_tree')

class ChannelSerializer(serializers.ModelSerializer):
    resource_count = serializers.SerializerMethodField('count_resources')
    resource_size = serializers.SerializerMethodField('calculate_resources_size')

    def count_resources(self, channel):
        if not channel.main_tree:
            return 0
        else:
            return count_files(channel.main_tree)

    def calculate_resources_size(self, channel):
        if not channel.main_tree:
            return 0
        else:
            return get_total_size(channel.main_tree)

    class Meta:
        model = Channel
        fields = ('id', 'name', 'description', 'editors', 'main_tree',
                    'clipboard_tree', 'trash_tree','resource_count', 'resource_size',
                    'version', 'thumbnail', 'deleted', 'public', 'pending_editors')

class FileSerializer(serializers.ModelSerializer):
    file_on_disk = serializers.SerializerMethodField('get_file_url')
    recommended_kind = serializers.SerializerMethodField('retrieve_recommended_kind')
    mimetype = serializers.SerializerMethodField('retrieve_extension')

    def get(*args, **kwargs):
         return super.get(*args, **kwargs)
    class Meta:
        model = File
        fields = ('id', 'checksum', 'file_size', 'file_on_disk', 'contentnode', 'file_format', 'preset', 'original_filename','recommended_kind', 'mimetype')

    def get_file_url(self, obj):
        return obj.file_on_disk.url

    def retrieve_recommended_kind(self, obj):
        return FormatPreset.objects.filter(allowed_formats__extension__contains=obj.file_format).first().kind.pk

    def retrieve_extension(self, obj):
        return obj.file_format.mimetype

class FileFormatSerializer(serializers.ModelSerializer):
    class Meta:
        model = FileFormat
        fields = ("__all__")

class FormatPresetSerializer(serializers.ModelSerializer):
   # files = FileSerializer(many=True, read_only=True)
    associated_mimetypes = serializers.SerializerMethodField('retrieve_mimetypes')

    def retrieve_mimetypes(self, preset):
        mimetypes = []
        for m in preset.allowed_formats.all():
            mimetypes.append(m.mimetype)
        return mimetypes

    class Meta:
        model = FormatPreset
        fields = ('id', 'readable_name', 'multi_language', 'supplementary', 'order', 'kind', 'allowed_formats','associated_mimetypes')

class ContentKindSerializer(serializers.ModelSerializer):
    associated_presets = serializers.SerializerMethodField('retrieve_associated_presets')
    def retrieve_associated_presets(self, kind):
        return FormatPreset.objects.filter(kind = kind).values()

    class Meta:
        model = ContentKind
        fields = ("kind", 'associated_presets')

class CustomListSerializer(serializers.ListSerializer):
    def update(self, instance, validated_data):
        node_mapping = {node.id: node for node in instance}
        update_nodes = {}
        tag_mapping = {}
        ret = []
        unformatted_input_tags = []

        with transaction.atomic():
            for item in validated_data:
                item_tags = item.get('tags')
                unformatted_input_tags += item.pop('tags')
                if 'id' in item:
                    update_nodes[item['id']] = item
                    tag_mapping[item['id']] = item_tags
                else:
                    # create new nodes
                    ret.append(ContentNode.objects.create(**item))

        # get all ContentTag objects, if doesn't exist, create them.
        all_tags = []

        for tag_data in unformatted_input_tags:
            # when deleting nodes, tag_data is a dict, but when adding nodes, it's a unicode string
            if isinstance(tag_data, unicode):
                tag_data = json.loads(tag_data)
            tag_tuple = ContentTag.objects.get_or_create(tag_name=tag_data['tag_name'], channel_id=tag_data['channel'])
            all_tags.append(tag_tuple[0])

        if ret:
            # new nodes and tags have been created, now add tags to them
            bulk_adding_list = []
            ThroughModel = ContentNode.tags.through
            for tag in all_tags:
                for node in ret:
                    bulk_adding_list.append(ThroughModel(node_id=node.pk, contenttag_id=tag.pk))
            ThroughModel.objects.bulk_create(bulk_adding_list)

        # Perform updates.
        if update_nodes:
            for node_id, data in update_nodes.items():
                node = node_mapping.get(node_id, None)
                if node:
                    # potential optimization opportunity
                    for attr, value in data.items():
                        setattr(node, attr, value)
                    taglist = []
                    for tag_data in tag_mapping.get(node_id, None):
                        # when deleting nodes, tag_data is a dict, but when adding nodes, it's a unicode string
                        if isinstance(tag_data, unicode):
                            tag_data = json.loads(tag_data)
{'fraction_non_alphanumeric': 0.08919244676576939, 'fraction_numerical': 0.0004821213338690237, 'mean_word_length': 2.8449181340747605, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6193572', 'n_tokens_mistral': 3388, 'n_tokens_neox': 3281, 'n_words': 789}