<commit_before># Copyright (c) The PyAMF Project.
# See LICENSE.txt for details.

"""
C-extension for L{pyamf.amf3} Python module in L{PyAMF<pyamf>}.

:since: 0.6
"""

from python cimport *

cdef extern from "datetime.h":
    void PyDateTime_IMPORT()
    int PyDateTime_Check(object)
    int PyTime_Check(object)

cdef extern from "math.h":
    float floor(float)

cdef extern from "stdlib.h" nogil:
    ctypedef unsigned long size_t

    int memcmp(void *dest, void *src, size_t)
    void *memcpy(void *, void *, size_t)

cdef extern from "Python.h":
    PyObject* Py_True
    PyObject *Py_None

    bint PyClass_Check(object)
    bint PyType_CheckExact(object)


from cpyamf cimport codec, amf3, util
import pyamf
from pyamf import util
import types


cdef char TYPE_NUMBER      = '\x00'
cdef char TYPE_BOOL        = '\x01'
cdef char TYPE_STRING      = '\x02'
cdef char TYPE_OBJECT      = '\x03'
cdef char TYPE_MOVIECLIP   = '\x04'
cdef char TYPE_NULL        = '\x05'
cdef char TYPE_UNDEFINED   = '\x06'
cdef char TYPE_REFERENCE   = '\x07'
cdef char TYPE_MIXEDARRAY  = '\x08'
cdef char TYPE_OBJECTTERM  = '\x09'
cdef char TYPE_ARRAY       = '\x0A'
cdef char TYPE_DATE        = '\x0B'
cdef char TYPE_LONGSTRING  = '\x0C'
cdef char TYPE_UNSUPPORTED = '\x0D'
cdef char TYPE_RECORDSET   = '\x0E'
cdef char TYPE_XML         = '\x0F'
cdef char TYPE_TYPEDOBJECT = '\x10'
cdef char TYPE_AMF3        = '\x11'

cdef PyObject *Undefined = <PyObject *>pyamf.Undefined
cdef PyObject *BuiltinFunctionType = <PyObject *>types.BuiltinFunctionType
cdef PyObject *GeneratorType = <PyObject *>types.GeneratorType
cdef object empty_string = str('')

PyDateTime_IMPORT


cdef class Context(codec.Context):
    """
    I hold the AMF0 context for en/decoding streams.
    """

    cdef list amf3_objs

    cpdef int clear(self) except? -1:
        """
        Clears the context.
        """
        codec.Context.clear(self)

        self.amf3_objs = []

    cpdef object hasAMF3ObjectReference(self, obj):
        """
        Gets a reference for an object.
        """
        return obj in self.amf3_objs

    cpdef int addAMF3Object(self, obj) except -1:
        """
        Adds an AMF3 reference to C{obj}.

        @type obj: C{mixed}
        @param obj: The object to add to the context.
        @rtype: C{int}
        @return: Reference to C{obj}.
        """
        return self.amf3_objs.append(obj)


cdef class Decoder(codec.Codec):
    """
    Decodes an AMF0 stream.
    """

    cdef Context buildContext(self):
        return Context()

    cdef object readNumber(self):
        """
        Reads a ActionScript C{Number} value.

        In ActionScript 1 and 2 the C{NumberASTypes} type represents all numbers,
        both floats and integers.

        @rtype: C{int} or C{float}
        """
        cdef double number
        cdef int done = 0
        cdef unsigned char *buf

        self.stream.read_double(&number)
        buf = <unsigned char *>&number

        if floor(number) == number:
            if memcmp(buf, &util.system_nan, 8) == 0:
                done = 1
            elif memcmp(buf, &util.system_posinf, 8) == 0:
                done = 1
            elif memcmp(buf, &util.system_neginf, 8) == 0:
                done = 1

            if done == 0:
                return int(number)

        return number

    def readBoolean(self):
        """
        Reads a ActionScript C{Boolean} value.

        @rtype: C{bool}
        @return: Boolean.
        """
        return bool(self.stream.read_uchar())

    def readNull(self):
        """
        Reads a ActionScript C{null} value.

        @return: C{None}
        @rtype: C{None}
        """
        return None

    def readUndefined(self):
        """
        Reads an ActionScript C{undefined} value.

        @return: L{Undefined<pyamf.Undefined>}
        """
        return pyamf.Undefined

    def readMixedArray(self):
        """
        Read mixed array.

        @rtype: C{dict}
        @return: C{dict} read from the stream
        """
        self.stream.read_ulong()
        obj = pyamf.MixedArray()
        self.context.addObject(obj)
        self._readObject(obj)
        ikeys = []

        for key in obj.keys():
            try:
                ikey = int(key)
                ikeys.append((key, ikey))
                obj[ikey] = obj[key]
                del obj[key]
            except ValueError:
                # XXX: do we want to ignore this?
                pass

        ikeys.sort()

        return obj

    def readList(self):
        """
        Read a C{list} from the data stream.

        @rtype: C{list}
        @return: C{list}
        """
        obj = []
        self.context.addObject(obj)
        len = self.stream.read_ulong()

        for i in xrange(len):
            obj.append(self.readElement())

        return obj

    def readTypedObject(self):
        """
        Reads an ActionScript object from the stream and attempts to
        'cast' it.

        @see: L{load_class<pyamf.load_class>}
        """
        classname = self.readString()
        alias = None

        try:
            alias = pyamf.get_class_alias(classname)

            ret = alias.createInstance(codec=self)
        except pyamf.UnknownClassAlias:
            if self.strict:
                raise

            ret = pyamf.TypedObject(classname)

        self.context.addObject(ret)
        self._readObject(ret, alias)

        return ret

    def _getAMF3Decoder(self):
        decoder = getattr(self, 'amf3_decoder', None)

        if not decoder:
            decoder = pyamf.get_decoder(pyamf.AMF3, stream=self.stream)

        return decoder

    def readAMF3(self):
        """
        Read AMF3 elements from the data stream.

        @rtype: C{mixed}
        @return: The AMF3 element read from the stream
        """
        decoder = self._getAMF3Decoder()

        element = decoder.readElement()
        self.context.addAMF3Object(element)

        return element

    def readString(self):
        """
        Reads a C{string} from the stream.
        """
        l = self.stream.read_ushort()

        return self.stream.read(l)

    cdef object readUnicode(self):
        """
        Reads a C{unicode} from the data stream.
        """
        cdef unsigned short l
        cdef char *bytes = NULL

        self.stream.read_ushort(&l)

        try:
            self.stream.read(&bytes, l)
        except:
            if bytes != NULL:
                PyMem_Free(bytes)

        return self.context.getUnicodeForString(bytes)

    cdef int _readObject(self, obj, alias=None) except -1:
        cdef dictobj_attrs = dict()

        key = self.readString()

        while self.stream.peek() != TYPE_OBJECTTERM:
            obj_attrs[key] = self.readElement()
            key = self.readString()

        # discard the end marker (TYPE_OBJECTTERM)
        self.stream.read(1)
<commit_msg>Move more AMF0 Decoder functions into c-equivalents.<commit_after># Copyright (c) The PyAMF Project.
# See LICENSE.txt for details.

"""
C-extension for L{pyamf.amf3} Python module in L{PyAMF<pyamf>}.

:since: 0.6
"""

from python cimport *

cdef extern from "datetime.h":
    void PyDateTime_IMPORT()
    int PyDateTime_Check(object)
    int PyTime_Check(object)

cdef extern from "math.h":
    float floor(float)

cdef extern from "stdlib.h" nogil:
    ctypedef unsigned long size_t

    int memcmp(void *dest, void *src, size_t)
    void *memcpy(void *, void *, size_t)

cdef extern from "Python.h":
    PyObject* Py_True
    PyObject *Py_None

    bint PyClass_Check(object)
    bint PyType_CheckExact(object)


from cpyamf cimport codec, amf3, util
import pyamf
import types


cdef char TYPE_NUMBER      = '\x00'
cdef char TYPE_BOOL        = '\x01'
cdef char TYPE_STRING      = '\x02'
cdef char TYPE_OBJECT      = '\x03'
cdef char TYPE_MOVIECLIP   = '\x04'
cdef char TYPE_NULL        = '\x05'
cdef char TYPE_UNDEFINED   = '\x06'
cdef char TYPE_REFERENCE   = '\x07'
cdef char TYPE_MIXEDARRAY  = '\x08'
cdef char TYPE_OBJECTTERM  = '\x09'
cdef char TYPE_ARRAY       = '\x0A'
cdef char TYPE_DATE        = '\x0B'
cdef char TYPE_LONGSTRING  = '\x0C'
cdef char TYPE_UNSUPPORTED = '\x0D'
cdef char TYPE_RECORDSET   = '\x0E'
cdef char TYPE_XML         = '\x0F'
cdef char TYPE_TYPEDOBJECT = '\x10'
cdef char TYPE_AMF3        = '\x11'

cdef PyObject *Undefined = <PyObject *>pyamf.Undefined
cdef PyObject *BuiltinFunctionType = <PyObject *>types.BuiltinFunctionType
cdef PyObject *GeneratorType = <PyObject *>types.GeneratorType
cdef object empty_string = str('')

PyDateTime_IMPORT


cdef class Context(codec.Context):
    """
    I hold the AMF0 context for en/decoding streams.
    """

    cdef list amf3_objs

    cpdef int clear(self) except? -1:
        """
        Clears the context.
        """
        codec.Context.clear(self)

        self.amf3_objs = []

    cdef object hasAMF3ObjectReference(self, obj):
        """
        Gets a reference for an object.
        """
        return obj in self.amf3_objs

    cdef int addAMF3Object(self, obj) except -1:
        """
        Adds an AMF3 reference to C{obj}.

        @type obj: C{mixed}
        @param obj: The object to add to the context.
        @rtype: C{int}
        @return: Reference to C{obj}.
        """
        return self.amf3_objs.append(obj)


cdef class Decoder(codec.Codec):
    """
    Decodes an AMF0 stream.
    """

    cdef amf3.Decoder amf3_decoder

    cef __cinit__(self):
        self.amf3_decoder = NULL

    cdef Context buildContext(self):
        return Context()

    cdef object readNumber(self):
        """
        Reads a ActionScript C{Number} value.

        In ActionScript 1 and 2 the C{NumberASTypes} type represents all numbers,
        both floats and integers.

        @rtype: C{int} or C{float}
        """
        cdef double number
        cdef int done = 0
        cdef unsigned char *buf

        self.stream.read_double(&number)
        #buf = <unsigned char *>&number

        #if floor(number) == number:
        #    if memcmp(buf, &util.system_nan, 8) == 0:
        #        done = 1
        #    elif memcmp(buf, &util.system_posinf, 8) == 0:
        #        done = 1
        #    elif memcmp(buf, &util.system_neginf, 8) == 0:
        #        done = 1

        #    if done == 0:
        #        return int(number)

        return number

    cdef object readBoolean(self):
        cdef unsigned char b

        self.stream.read_uchar(&b)

        if b == 1:
            return True
        elif b == 0:
            return False

        raise pyamf.DecodeError('Unexpected value when decoding boolean')

    cdef inline object readNull(self):
        return None

    cdef inline object readUndefined(self):
        return <object>Undefined

    def readMixedArray(self):
        """
        Read mixed array.

        @rtype: C{dict}
        @return: C{dict} read from the stream
        """
        cdef unsigned long l

        self.stream.read_ulong(&l)

        obj = pyamf.MixedArray()
        self.context.addObject(obj)
        self._readObject(obj)
        ikeys = []

        for key in obj.keys():
            try:
                ikey = int(key)
                ikeys.append((key, ikey))
                obj[ikey] = obj[key]
                del obj[key]
            except ValueError:
                # XXX: do we want to ignore this?
                pass

        ikeys.sort()

        return obj

    cdef object readList(self):
        """
        Read a C{list} from the data stream.

        @rtype: C{list}
        @return: C{list}
        """
        cdef unsigned long l
        cdef list obj = []

        self.context.addObject(obj)
        self.stream.read_ulong(&l)

        for 0 <= i < l:
            PyList_Append(obj, self.readElement())

        return obj

    def readTypedObject(self):
        """
        Reads an ActionScript object from the stream and attempts to
        'cast' it.

        @see: L{load_class<pyamf.load_class>}
        """
        classname = self.readString()
        alias = None

        try:
            alias = pyamf.get_class_alias(classname)

            ret = alias.createInstance(codec=self)
        except pyamf.UnknownClassAlias:
            if self.strict:
                raise

            ret = pyamf.TypedObject(classname)

        self.context.addObject(ret)
        self._readObject(ret, alias)

        return ret

    cdef inline amf3.Decoder _getAMF3Decoder(self):
        if self.amf3_decoder == NULL:
            self.amf3_decoder = amf3.Decoder(self.stream)

        return self.amf3_decoder

    cdef object readAMF3(self):
        """
        Read AMF3 elements from the data stream.

        @rtype: C{mixed}
        @return: The AMF3 element read from the stream
        """
        cdef amf3.Decoder decoder = self._getAMF3Decoder()
        cdef object element = decoder.readElement()

        self.context.addAMF3Object(element)

        return element

    cdef inline char *readBytes(self, bint big=0) except NULL:
        cdef Py_ssize_t l
        char *buf = NULL

        if not big:
            self.stream.read_ushort(&<unsigned short>l)
        else:
            self.stream.read_ulong(&<unsigned long>l)

        self.stream.read(buf, l)

        return buf

    cdef object readString(self, bint big=0):
        """
        Reads a C{string} from the stream.
        """
        cdef char *buf = self.readBytes(big)
        cdef object s

        try:
            s = PyString_FromStringAndSize(buf, r)
        finally:
            if buf != NULL:
                free(buf)

        return s

    cdef object readUnicode(self, bint big=0):
        """
        Reads a C{unicode} from the data stream.
        """
        cdef char *bytes = self.readBytes(big)

        try:
            return self.context.getUnicodeForString(bytes)
        finally:
            free(bytes)

    cdef int _readObject(self, obj, alias=None) except -1:
        cdef dict obj_attrs = {}
        cdef char *key

        while self.stream.peek() != TYPE_OBJECTTERM:
            key = self.readBytes()
            # change to PyDict_SetString()
            obj_attrs[key] = self.readElement()
            free(key)

        # discard the end marker (TYPE_OBJECTTERM)
        self.stream.read(1)

{'fraction_non_alphanumeric': 0.09857182861943434, 'fraction_numerical': 0.010641276953234389, 'mean_word_length': 1.9141166870665034, 'pattern_counts': {'":': 6, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7080658', 'n_tokens_mistral': 4669, 'n_tokens_neox': 4460, 'n_words': 1221}