<commit_before>from crashmanager.models import CrashEntry, Bucket, Platform, Product, OS, TestCase, Client, Tool
from rest_framework import serializers
from django.forms import widgets
from django.core.exceptions import MultipleObjectsReturned
from FTB.Signatures.CrashInfo import CrashInfo
from FTB.ProgramConfiguration import ProgramConfiguration

from django.core.files.base import ContentFile
import hashlib
import base64

class CrashEntrySerializer(serializers.ModelSerializer):
    # We need to redefine several fields explicitly because we flatten our
    # foreign keys into these fields instead of using primary keys, hyperlinks
    # or slug fields. All of the other solutions would require the client to
    # create these instances first and issue multiple requests in total.
    platform = serializers.CharField(max_length=63)
    product = serializers.CharField(max_length=63)
    product_version = serializers.CharField(max_length=63, required=False, write_only=True)
    os = serializers.CharField(max_length=63)
    client = serializers.CharField(max_length=255)
    tool = serializers.CharField(max_length=63)
    testcase = serializers.CharField(widget=widgets.Textarea, required=False)
    testcase_ext = serializers.CharField(required=False, write_only=True)
    testcase_quality = serializers.CharField(required=False, default=0, write_only=True)
    testcase_isbinary = serializers.BooleanField(required=False, default=False, write_only=True)

    class Meta:
        model = CrashEntry
        fields = (
                  'rawStdout', 'rawStderr', 'rawCrashData', 'metadata', 
                  'testcase', 'testcase_ext', 'testcase_quality', 'testcase_isbinary',
                  'platform', 'product', 'product_version', 'os', 'client', 'tool', 
                  'env', 'args'
                  )

    def to_native(self, obj):
        '''
        Serialize (flatten) our object. We need custom flattening because we
        want the foreign relationships of our object to be flattened into our
        object by name. Furthermore, we display the testcase here if it is
        not binary.
        '''
        serialized = super(CrashEntrySerializer, self).to_native(obj)
        if obj != None:
            serialized["product"] = obj.product.name
            if obj.product.version:
                serialized["product_version"] = obj.product.version
                
            serialized["os"] = obj.os.name
            serialized["platform"] = obj.platform.name
            serialized["client"] = obj.client.name
            serialized["tool"] = obj.tool.name
            
            if obj.testcase:
                serialized["testcase_isbinary"] = obj.testcase.isBinary
                serialized["testcase_quality"] = obj.testcase.quality
                serialized["testcase"] = str(obj.testcase.test)
        
        return serialized

    def restore_object(self, attrs, instance=None):
        '''
        Create a CrashEntry instance based on the given dictionary of values
        received. We need to unflatten foreign relationships like product,
        platform, os and client and create the foreign objects on the fly
        if they don't exist in our database yet.
        '''
        if instance:
            # Not allowed to update existing instances
            return instance
        
        product = attrs.pop('product', None)
        product_version = attrs.pop('product_version', None)
        platform = attrs.pop('platform', None)
        os = attrs.pop('os', None)
        client = attrs.pop('client', None)
        tool = attrs.pop('tool', None)
        testcase = attrs.pop('testcase', None)
        testcase_ext = attrs.pop('testcase_ext', None)
        testcase_quality = attrs.pop('testcase_quality', 0)
        testcase_isbinary = attrs.pop('testcase_isbinary', False)
        
        # Parse the incoming data using the crash signature package from FTB
        configuration = ProgramConfiguration(product, platform, os, product_version)
        crashInfo = CrashInfo.fromRawCrashData(attrs['rawStdout'], attrs['rawStderr'], configuration, attrs['rawCrashData'])
        
        # Populate certain fields here from the CrashInfo object we just got
        if crashInfo.crashAddress != None:
            attrs['crashAddress'] = hex(crashInfo.crashAddress)
        attrs['shortSignature'] = crashInfo.createShortSignature()
        
        def createOrGetModelByName(model, attrs):
            '''
            Generically determine if the given model with the given attributes
            already exists in our database. If so, return that object, otherwise
            create it on the fly.
            
            @type model: Class
            @param model: The model to use for filtering and instantiating
            
            @type attrs: dict
            @param attrs: Dictionary of attributes to use for filtering/instantiating
            
            @rtype: model
            @return The model instance
            '''
            objs = model.objects.filter(**attrs)
                  
            if len(objs) > 1:
                raise MultipleObjectsReturned("Multiple objects with same keyword combination in database!")
        
            if len(objs) == 0:
                dbobj = model(**attrs)
                dbobj.save()
                return dbobj
            else:
                return objs.first()
        
        # Get or instantiate objects for product, platform, os, client and tool
        attrs['product'] = createOrGetModelByName(Product, { 'name' : product, 'version' : product_version })
        attrs['platform'] = createOrGetModelByName(Platform, { 'name' : platform })
        attrs['os'] = createOrGetModelByName(OS, { 'name' : os })
        attrs['client'] = createOrGetModelByName(Client, { 'name' : client })
        attrs['tool'] = createOrGetModelByName(Tool, { 'name' : tool })

        # If a testcase is supplied, create a testcase object and store it
        if testcase:
            if testcase_ext == None:
                raise RuntimeError("Must provide testcase extension when providing testcase")
            
            if testcase_isbinary:
                testcase = base64.b64decode(testcase)
            
            h = hashlib.new('sha1')
            if testcase_isbinary:
                h.update(str(testcase))
            else:
                h.update(repr(testcase))

            dbobj = TestCase(quality=testcase_quality, isBinary=testcase_isbinary, size=len(testcase))
            dbobj.test.save("%s.%s" % (h.hexdigest(), testcase_ext), ContentFile(testcase))
            dbobj.save()
            attrs['testcase'] = dbobj
        else:
            attrs['testcase'] = None
        
        # Create our CrashEntry instance
        return super(CrashEntrySerializer, self).restore_object(attrs, instance)<commit_msg>Style fixes<commit_after>import base64
from django.core.exceptions import MultipleObjectsReturned
from django.core.files.base import ContentFile
from django.forms import widgets
import hashlib
from rest_framework import serializers

from FTB.ProgramConfiguration import ProgramConfiguration
from FTB.Signatures.CrashInfo import CrashInfo
from crashmanager.models import CrashEntry, Bucket, Platform, Product, OS, TestCase, Client, Tool


class CrashEntrySerializer(serializers.ModelSerializer):
    # We need to redefine several fields explicitly because we flatten our
    # foreign keys into these fields instead of using primary keys, hyperlinks
    # or slug fields. All of the other solutions would require the client to
    # create these instances first and issue multiple requests in total.
    platform = serializers.CharField(max_length=63)
    product = serializers.CharField(max_length=63)
    product_version = serializers.CharField(max_length=63, required=False, write_only=True)
    os = serializers.CharField(max_length=63)
    client = serializers.CharField(max_length=255)
    tool = serializers.CharField(max_length=63)
    testcase = serializers.CharField(widget=widgets.Textarea, required=False)
    testcase_ext = serializers.CharField(required=False, write_only=True)
    testcase_quality = serializers.CharField(required=False, default=0, write_only=True)
    testcase_isbinary = serializers.BooleanField(required=False, default=False, write_only=True)

    class Meta:
        model = CrashEntry
        fields = (
                  'rawStdout', 'rawStderr', 'rawCrashData', 'metadata',
                  'testcase', 'testcase_ext', 'testcase_quality', 'testcase_isbinary',
                  'platform', 'product', 'product_version', 'os', 'client', 'tool',
                  'env', 'args'
                  )

    def to_native(self, obj):
        '''
        Serialize (flatten) our object. We need custom flattening because we
        want the foreign relationships of our object to be flattened into our
        object by name. Furthermore, we display the testcase here if it is
        not binary.
        '''
        serialized = super(CrashEntrySerializer, self).to_native(obj)
        if obj != None:
            serialized["product"] = obj.product.name
            if obj.product.version:
                serialized["product_version"] = obj.product.version

            serialized["os"] = obj.os.name
            serialized["platform"] = obj.platform.name
            serialized["client"] = obj.client.name
            serialized["tool"] = obj.tool.name

            if obj.testcase:
                serialized["testcase_isbinary"] = obj.testcase.isBinary
                serialized["testcase_quality"] = obj.testcase.quality
                serialized["testcase"] = str(obj.testcase.test)

        return serialized

    def restore_object(self, attrs, instance=None):
        '''
        Create a CrashEntry instance based on the given dictionary of values
        received. We need to unflatten foreign relationships like product,
        platform, os and client and create the foreign objects on the fly
        if they don't exist in our database yet.
        '''
        if instance:
            # Not allowed to update existing instances
            return instance

        product = attrs.pop('product', None)
        product_version = attrs.pop('product_version', None)
        platform = attrs.pop('platform', None)
        os = attrs.pop('os', None)
        client = attrs.pop('client', None)
        tool = attrs.pop('tool', None)
        testcase = attrs.pop('testcase', None)
        testcase_ext = attrs.pop('testcase_ext', None)
        testcase_quality = attrs.pop('testcase_quality', 0)
        testcase_isbinary = attrs.pop('testcase_isbinary', False)

        # Parse the incoming data using the crash signature package from FTB
        configuration = ProgramConfiguration(product, platform, os, product_version)
        crashInfo = CrashInfo.fromRawCrashData(attrs['rawStdout'], attrs['rawStderr'], configuration, attrs['rawCrashData'])

        # Populate certain fields here from the CrashInfo object we just got
        if crashInfo.crashAddress != None:
            attrs['crashAddress'] = hex(crashInfo.crashAddress)
        attrs['shortSignature'] = crashInfo.createShortSignature()

        def createOrGetModelByName(model, attrs):
            '''
            Generically determine if the given model with the given attributes
            already exists in our database. If so, return that object, otherwise
            create it on the fly.
            
            @type model: Class
            @param model: The model to use for filtering and instantiating
            
            @type attrs: dict
            @param attrs: Dictionary of attributes to use for filtering/instantiating
            
            @rtype: model
            @return The model instance
            '''
            objs = model.objects.filter(**attrs)

            if len(objs) > 1:
                raise MultipleObjectsReturned("Multiple objects with same keyword combination in database!")

            if len(objs) == 0:
                dbobj = model(**attrs)
                dbobj.save()
                return dbobj
            else:
                return objs.first()

        # Get or instantiate objects for product, platform, os, client and tool
        attrs['product'] = createOrGetModelByName(Product, { 'name' : product, 'version' : product_version })
        attrs['platform'] = createOrGetModelByName(Platform, { 'name' : platform })
        attrs['os'] = createOrGetModelByName(OS, { 'name' : os })
        attrs['client'] = createOrGetModelByName(Client, { 'name' : client })
        attrs['tool'] = createOrGetModelByName(Tool, { 'name' : tool })

        # If a testcase is supplied, create a testcase object and store it
        if testcase:
            if testcase_ext == None:
                raise RuntimeError("Must provide testcase extension when providing testcase")

            if testcase_isbinary:
                testcase = base64.b64decode(testcase)

            h = hashlib.new('sha1')
            if testcase_isbinary:
                h.update(str(testcase))
            else:
                h.update(repr(testcase))

            dbobj = TestCase(quality=testcase_quality, isBinary=testcase_isbinary, size=len(testcase))
            dbobj.test.save("%s.%s" % (h.hexdigest(), testcase_ext), ContentFile(testcase))
            dbobj.save()
            attrs['testcase'] = dbobj
        else:
            attrs['testcase'] = None

        # Create our CrashEntry instance
        return super(CrashEntrySerializer, self).restore_object(attrs, instance)
{'fraction_non_alphanumeric': 0.08794764320905948, 'fraction_numerical': 0.003529671299360247, 'mean_word_length': 2.4791506779227426, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4719963', 'n_tokens_mistral': 3534, 'n_tokens_neox': 3474, 'n_words': 1006}