<commit_before>#  Copyright (c) 2012-2017 <NAME>
#
#  This file is part of VapourSynth.
#
#  VapourSynth is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2.1 of the License, or (at your option) any later version.
#
#  VapourSynth is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with VapourSynth; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
""" This is the VapourSynth module implementing the Python bindings. """

cimport vapoursynth
cimport cython.parallel
from cython cimport view
from libc.stdint cimport intptr_t, uint16_t, uint32_t
from cpython.buffer cimport (PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_STRIDES,
                             PyBUF_F_CONTIGUOUS)
from cpython.ref cimport Py_INCREF, Py_DECREF
import os
import ctypes
import threading
import traceback
import gc
import sys
import inspect
from collections.abc import Iterable, Mapping
from fractions import Fraction

# Ensure that the import doesn't fail
# if typing is not available on the python installation.
try:
    import typing
except ImportError as e:
    typing = None


_using_vsscript = False
_environment_id_stack = []
_environment_id = None
_stored_outputs = {}
_cores = {}
_stored_output = {}
_core = None
_message_handler = None
cdef const VSAPI *_vsapi = NULL


# Create an empty list whose instance will represent a not passed value.
_EMPTY = []

def _construct_parameter(signature):
    name,type,*opt = signature.split(":")

    # Handle Arrays.
    if type.endswith("[]"):
        array = True
        type = type[:-2]
    else:
        array = False

    # Handle types
    if type == "clip":
        type = vapoursynth.VideoNode
    elif type == "frame":
        type = vapoursynth.VideoFrame
    elif type == "func":
        type = typing.Union[vapoursynth.Func, typing.Callable]
    elif type == "int":
        type = int
    elif type == "float":
        type = float
    elif type == "data":
        type = typing.Union[str, bytes, bytearray]
    else:
        raise ValueError("Couldn't determine type")

    # Make the type a sequence.
    if array:
        type = typing.Sequence[type]

    # Mark an optional type
    if opt:
        type = typing.Optional[type]
        opt = None
    else:
        opt = inspect.Parameter.empty
        
        
    return inspect.Parameter(
        name, inspect.Parameter.POSITIONAL_OR_KEYWORD,
        default=opt, annotation=type
    )

def construct_signature(signature, injected=None):
    if typing is None:
        raise RuntimeError("At least Python 3.5 is required to use type-hinting")
    
    if isinstance(signature, vapoursynth.Function):
        signature = signature.signature

    params = list(
        _construct_parameter(param)
        for param in signature.split(";")
        if param
    )
    
    if injected:
        del params[0]
    
    return inspect.Signature(tuple(params), return_annotation=vapoursynth.VideoNode)
    

class Error(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return str(self.value)
        
    def __repr__(self):
        return repr(self.value)

cdef void __stdcall message_handler_wrapper(int msgType, const char *msg, void *userData) nogil:
    with gil:
        global _message_handler
        _message_handler(msgType, msg.decode('utf-8'))
        
def set_message_handler(handler_func):
    cdef const VSAPI *funcs
    global _message_handler
    funcs = getVapourSynthAPI(VAPOURSYNTH_API_VERSION)
    if funcs == NULL:
        raise Error('Failed to obtain VapourSynth API pointer. Is the Python module and loaded core library mismatched?')
    if handler_func is None:
        _message_handler = None
        funcs.setMessageHandler(NULL, NULL)
    else:
        handler_func(vapoursynth.mtDebug, 'New message handler installed from python')
        _message_handler = handler_func
        funcs.setMessageHandler(message_handler_wrapper, NULL)
    
def _get_output_dict(funcname="this function"):
    global _using_vsscript
    if _using_vsscript:
        global _stored_outputs
        global _environment_id
        if _environment_id is None:
            raise Error('Internal environment id not set. %s called from a filter callback?'%funcname)
            
        return _stored_outputs[_environment_id]
    else:
        return _stored_output
    
def clear_output(int index = 0):
    cdef dict outputs = _get_output_dict("clear_output")
    try:
        del outputs[index]
    except KeyError:
        pass

def clear_outputs():
    cdef dict outputs = _get_output_dict("clear_outputs")
    outputs.clear()

def get_output(int index = 0):
    return _get_output_dict("get_output")[index]

cdef class FuncData(object):
    cdef object func
    cdef Core core
    cdef int id
    
    def __init__(self):
        raise Error('Class cannot be instantiated directly')
        
    def __call__(self, **kwargs):
        return self.func(**kwargs)

cdef FuncData createFuncData(object func, Core core, int id):
    cdef FuncData instance = FuncData.__new__(FuncData)
    instance.func = func
    instance.core = core
    instance.id = id
    return instance
    
cdef class Func(object):
    cdef const VSAPI *funcs
    cdef VSFuncRef *ref
    
    def __init__(self):
        raise Error('Class cannot be instantiated directly')
        
    def __dealloc__(self):
        self.funcs.freeFunc(self.ref)
        
    def __call__(self, **kwargs):
        cdef VSMap *outm
        cdef VSMap *inm
        cdef const VSAPI *vsapi
        cdef const char *error
        vsapi = vpy_getVSApi();
        outm = self.funcs.createMap()
        inm = self.funcs.createMap()
        try:
            dictToMap(kwargs, inm, None, vsapi)
            self.funcs.callFunc(self.ref, inm, outm, NULL, NULL)
            error = self.funcs.getError(outm)
            if error:
                raise Error(error.decode('utf-8'))
            ret = mapToDict(outm, False, False, None, vsapi)
            if not isinstance(ret, dict):
                ret = {'val':ret}
        finally:
            vsapi.freeMap(outm)
            vsapi.freeMap(inm)
        
cdef Func createFuncPython(object func, Core core):
    cdef Func instance = Func.__new__(Func)
    instance.funcs = core.funcs
    if _using_vsscript:
        global _environment_id
        if _environment_id is None:
            raise Error('Internal environment id not set. Report this function wrapper creation error.')
        fdata = createFuncData(func, core, _environment_id)
    else:
        fdata = createFuncData(func, core, 0)
    Py_INCREF(fdata)
    instance.ref = instance.funcs.createFunc(publicFunction, <void *>fdata, freeFunc, core.core, core.funcs)
    return instance
        
cdef Func createFuncRef(VSFuncRef *ref, const VSAPI *funcs):
    cdef Func instance = Func.__new__(Func)
    instance.funcs = funcs
    instance.ref = ref
    return instance

cdef class RawCallbackData(object):
    cdef const VSAPI *funcs
    cdef object callback

    cdef VideoNode node

    cdef object wrap_cb
    cdef object future

    def __init__(self, VideoNode node, object callback = None):
        self.node = node
        self.callback = callback

        self.future = None
        self.wrap_cb = None

    def for_future(self, object future, object wrap_call=None):
        if wrap_call is None:
            wrap_call = lambda func, *args, **kwargs: func(*args, **kwargs)
        self.callback = self.handle_future
        self.future = future
        self.wrap_cb = wrap_call

    def handle_future(self, node, n, result):
        if isinstance(result, Error):
            func = self.future.set_exception
        else:
            func = self.future.set_result

        self.wrap_cb(func, result)

    def receive(self, n, result):
        self.callback(self.node, n, result)


cdef createRawCallbackData(const VSAPI* funcs, VideoNode videonode, object cb, object wrap_call=None):
    cbd = RawCallbackData(videonode, cb)
    if not callable(cb):
        cbd.for_future(cb, wrap_call)
    cbd.funcs = funcs
    return cbd

cdef class CallbackData(object):
    cdef VideoNode node
    cdef const VSAPI *funcs
    cdef object fileobj
    cdef int output
    cdef int requested
    cdef int completed
    cdef int total
    cdef int num_planes
    cdef bint y4m
    cdef dict reorder
    cdef object condition
    cdef object progress_update
    cdef str error

    def __init__(self, fileobj, requested, total, num_planes, y4m, node, progress_update):
        self.fileobj = fileobj
        self.output = 0
        self.requested = requested
        self.completed = 0
        self.total = total
        self.num_planes = num_planes
        self.y4m = y4m
        self.condition = threading.Condition()
        self.node = node
        self.progress_update = progress_update
        self.funcs = (<VideoNode>node).funcs
        self.reorder = {}


cdef class FramePtr(object):
    cdef const VSFrameRef *f
    cdef const VSAPI *funcs

    def __init__(self):
        raise Error('Class cannot be instantiated directly')

    def __dealloc__(self):
        self.funcs.freeFrame(self.f)

cdef FramePtr createFramePtr(const VSFrameRef *f, const VSAPI *funcs):
    cdef FramePtr instance = FramePtr.__new__(FramePtr)    
    instance.f = f
    instance.funcs = funcs
    return instance

cdef void __stdcall frameDoneCallbackRaw(void *data, const VSFrameRef *f, int n, VSNodeRef *node, const char *errormsg) nogil:
    with gil:
        d = <RawCallbackData>data
        if f == NULL:
            result = ''
            if errormsg != NULL:
                result = errormsg.decode('utf-8')
            result = Error(result)

        else:
            result = createConstVideoFrame(f, d.funcs, d.node.core)

        try:
            d.receive(n, result)
        except:
            import traceback
            traceback.print_exc()
        finally:
            Py_DECREF(d)


cdef void __stdcall frameDoneCallbackOutput(void *data, const VSFrameRef *f, int n, VSNodeRef *node, const char *errormsg) with gil:
    cdef VideoFrame frame_obj
    cdef VideoPlane plane
    cdef int x

    cdef CallbackData d = <CallbackData>data
    d.completed += 1

    if f == NULL:
        d.total = d.requested
        if errormsg == NULL:
            d.error = 'Failed to retrieve frame ' + str(n)
        else:
            d.error = 'Failed to retrieve frame ' + str(n) + ' with error: ' + errormsg.decode('utf-8')
        d.output += 1
    else:
        d.reorder[n] = createConstVideoFrame(f, d.funcs, d.node.core)

        while d.output in d.reorder:
            frame_obj = <VideoFrame>d.reorder[d.output]
            try:
                if d.y4m:
                    d.fileobj.write(b'FRAME\n')
                for x in range(frame_obj.format.num_planes):
                    plane = VideoPlane.__new__(VideoPlane, frame_obj, x)
                    d.fileobj.write(plane)
            except:
                d.error = 'File write call returned an error'
                d.total = d.requested

            del d.reorder[d.output]
            d.output += 1

        if d.progress_update is not None:
            try:
                d.progress_update(d.completed, d.total)
            except BaseException as e:
                d.error = 'Progress update caused an exception: ' + str(e)
                d.total = d.requested

    if d.requested < d.total:
        d.node.funcs.getFrameAsync(d.requested, d.node.node, frameDoneCallbackOutput, data)
        d.requested += 1

    d.condition.acquire()
    d.condition.notify()
    d.condition.release()


cdef object mapToDict(const VSMap *map, bint flatten, bint add_cache, Core core, const VSAPI *funcs):
    cdef int numKeys = funcs.propNumKeys(map)
    retdict = {}
    cdef const char *retkey
    cdef char proptype

    for x in range(numKeys):
        retkey = funcs.propGetKey(map, x)
        proptype = funcs.propGetType(map, retkey)

        for y in range(funcs.propNumElements(map, retkey)):
            if proptype == 'i':
                newval = funcs.propGetInt(map, retkey, y, NULL)
            elif proptype == 'f':
                newval = funcs.propGetFloat(map, retkey, y, NULL)
            elif proptype == 's':
                newval = funcs.propGetData(map, retkey, y, NULL)
            elif proptype =='c':
                newval = createVideoNode(funcs.propGetNode(map, retkey, y, NULL), funcs, core)

                if add_cache and not (newval.flags & vapoursynth.nfNoCache):
                    newval = core.std.Cache(clip=newval)

                    if isinstance(newval, dict):
                        newval = newval['dict']
            elif proptype =='v':
                newval = createConstVideoFrame(funcs.propGetFrame(map, retkey, y, NULL), funcs, core)
            elif proptype =='m':
                newval = createFuncRef(funcs.propGetFunc(map, retkey, y, NULL), funcs)

            if y == 0:
                vval = newval
            elif y == 1:
                vval = [vval, newval]
            else:
                vval.append(newval)
        retdict[retkey.decode('utf-8')] = vval

    if not flatten:
        return retdict
    elif len(retdict) == 0:
        return None
    elif len(retdict) == 1:
        a, b = retdict.popitem()
        return b
    else:
        return retdict

cdef void dictToMap(dict ndict, VSMap *inm, Core core, const VSAPI *funcs) except *:
    for key in ndict:
        ckey = key.encode('utf-8')
        val = ndict[key]

        if isinstance(val, (str, bytes, bytearray, VideoNode)):
            val = [val]
        else:
            try:
                iter(val)
            except:
                val = [val]     

        for v in val:
            if isinstance(v, VideoNode):
                if funcs.propSetNode(inm, ckey, (<VideoNode>v).node, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, VideoFrame):
                if funcs.propSetFrame(inm, ckey, (<VideoFrame>v).constf, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, Func):
                if funcs.propSetFunc(inm, ckey, (<Func>v).ref, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif callable(v):
                tf = createFuncPython(v, core)

                if funcs.propSetFunc(inm, ckey, tf.ref, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, int):
                if funcs.propSetInt(inm, ckey, int(v), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, float):
                if funcs.propSetFloat(inm, ckey, float(v), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, str):
                s = str(v).encode('utf-8')

                if funcs.propSetData(inm, ckey, s, -1, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, (bytes, bytearray)):
                if funcs.propSetData(inm, ckey, v, <int>len(v), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            else:
                raise Error('argument ' + key + ' was passed an unsupported type (' + type(v).__name__ + ')')


cdef void typedDictToMap(dict ndict, dict atypes, VSMap *inm, Core core, const VSAPI *funcs) except *:
    for key in ndict:
        ckey = key.encode('utf-8')
        val = ndict[key]
        if val is None:
            continue

        if isinstance(val, (str, bytes, bytearray, VideoNode)) or not isinstance(val, Iterable):
            val = [val]

        for v in val:
            if atypes[key][:4] == 'clip' and isinstance(v, VideoNode):
                if funcs.propSetNode(inm, ckey, (<VideoNode>v).node, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:5] == 'frame' and isinstance(v, VideoFrame):
                if funcs.propSetFrame(inm, ckey, (<VideoFrame>v).constf, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:4] == 'func' and isinstance(v, Func):
                if funcs.propSetFunc(inm, ckey, (<Func>v).ref, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:4] == 'func' and callable(v):
                tf = createFuncPython(v, core)
                if funcs.propSetFunc(inm, ckey, tf.ref, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:3] == 'int':
                if funcs.propSetInt(inm, ckey, int(v), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:5] == 'float':
                if funcs.propSetFloat(inm, ckey, float(v), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:4] == 'data':
                if not isinstance(v, (str, bytes, bytearray)):
                    v = str(v)
                if isinstance(v, str):
                    s = v.encode('utf-8')
                else:
                    s = v
                if funcs.propSetData(inm, ckey, s, <int>len(s), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            else:
                raise Error('argument ' + key + ' was passed an unsupported type (expected ' + atypes[key] + ' compatible type but got ' + type(v).__name__ + ')')
        if len(val) == 0:
        # set an empty key if it's an empty array
            if atypes[key][:4] == 'clip':
                funcs.propSetNode(inm, ckey, NULL, 2)
            elif atypes[key][:5] == 'frame':
                funcs.propSetFrame(inm, ckey, NULL, 2)
            elif atypes[key][:4] == 'func':
                funcs.propSetFunc(inm, ckey, NULL, 2)
            elif atypes[key][:3] == 'int':
                funcs.propSetInt(inm, ckey, 0, 2)
            elif atypes[key][:5] == 'float':
                funcs.propSetFloat(inm, ckey, 0, 2)
            elif atypes[key][:4] == 'data':
                funcs.propSetData(inm, ckey, NULL, 0, 2)
            else:
                raise Error('argument ' + key + ' has an unknown type: ' + atypes[key])

cdef class Format(object):
    cdef readonly int id
    cdef readonly str name
    cdef readonly object color_family
    cdef readonly object sample_type
    cdef readonly int bits_per_sample
    cdef readonly int bytes_per_sample
    cdef readonly int subsampling_w
    cdef readonly int subsampling_h
    cdef readonly int num_planes

    def __init__(self):
        raise Error('Class cannot be instantiated directly')

    def _as_dict(self):
        return {
            'color_family': self.color_family,
            'sample_type': self.sample_type,
            'bits_per_sample': self.bits_per_sample,
            'subsampling_w': self.subsampling_w,
            'subsampling_h': self.subsampling_h
        }

    def replace(self, **kwargs):
        core = kwargs.pop("core", None) or get_core()
        vals = self._as_dict()
        vals.update(**kwargs)
        return core.register_format(**vals)

    def __str__(self):
        return ('Format Descriptor\n'
               f'\tId: {self.id:d}\n'
               f'\tName: {self.name}\n'
               f'\tColor Family: {self.color_family.name}\n'<commit_msg>Allow to pass format objects to Filters<commit_after>#  Copyright (c) 2012-2017 <NAME>
#
#  This file is part of VapourSynth.
#
#  VapourSynth is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2.1 of the License, or (at your option) any later version.
#
#  VapourSynth is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with VapourSynth; if not, write to the Free Software
#  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
""" This is the VapourSynth module implementing the Python bindings. """

cimport vapoursynth
cimport cython.parallel
from cython cimport view
from libc.stdint cimport intptr_t, uint16_t, uint32_t
from cpython.buffer cimport (PyBUF_WRITABLE, PyBUF_FORMAT, PyBUF_STRIDES,
                             PyBUF_F_CONTIGUOUS)
from cpython.ref cimport Py_INCREF, Py_DECREF
import os
import ctypes
import threading
import traceback
import gc
import sys
import inspect
from collections.abc import Iterable, Mapping
from fractions import Fraction

# Ensure that the import doesn't fail
# if typing is not available on the python installation.
try:
    import typing
except ImportError as e:
    typing = None


_using_vsscript = False
_environment_id_stack = []
_environment_id = None
_stored_outputs = {}
_cores = {}
_stored_output = {}
_core = None
_message_handler = None
cdef const VSAPI *_vsapi = NULL


# Create an empty list whose instance will represent a not passed value.
_EMPTY = []

def _construct_parameter(signature):
    name,type,*opt = signature.split(":")

    # Handle Arrays.
    if type.endswith("[]"):
        array = True
        type = type[:-2]
    else:
        array = False

    # Handle types
    if type == "clip":
        type = vapoursynth.VideoNode
    elif type == "frame":
        type = vapoursynth.VideoFrame
    elif type == "func":
        type = typing.Union[vapoursynth.Func, typing.Callable]
    elif type == "int":
        type = int
    elif type == "float":
        type = float
    elif type == "data":
        type = typing.Union[str, bytes, bytearray]
    else:
        raise ValueError("Couldn't determine type")

    # Make the type a sequence.
    if array:
        type = typing.Sequence[type]

    # Mark an optional type
    if opt:
        type = typing.Optional[type]
        opt = None
    else:
        opt = inspect.Parameter.empty
        
        
    return inspect.Parameter(
        name, inspect.Parameter.POSITIONAL_OR_KEYWORD,
        default=opt, annotation=type
    )

def construct_signature(signature, injected=None):
    if typing is None:
        raise RuntimeError("At least Python 3.5 is required to use type-hinting")
    
    if isinstance(signature, vapoursynth.Function):
        signature = signature.signature

    params = list(
        _construct_parameter(param)
        for param in signature.split(";")
        if param
    )
    
    if injected:
        del params[0]
    
    return inspect.Signature(tuple(params), return_annotation=vapoursynth.VideoNode)
    

class Error(Exception):
    def __init__(self, value):
        self.value = value

    def __str__(self):
        return str(self.value)
        
    def __repr__(self):
        return repr(self.value)

cdef void __stdcall message_handler_wrapper(int msgType, const char *msg, void *userData) nogil:
    with gil:
        global _message_handler
        _message_handler(msgType, msg.decode('utf-8'))
        
def set_message_handler(handler_func):
    cdef const VSAPI *funcs
    global _message_handler
    funcs = getVapourSynthAPI(VAPOURSYNTH_API_VERSION)
    if funcs == NULL:
        raise Error('Failed to obtain VapourSynth API pointer. Is the Python module and loaded core library mismatched?')
    if handler_func is None:
        _message_handler = None
        funcs.setMessageHandler(NULL, NULL)
    else:
        handler_func(vapoursynth.mtDebug, 'New message handler installed from python')
        _message_handler = handler_func
        funcs.setMessageHandler(message_handler_wrapper, NULL)
    
def _get_output_dict(funcname="this function"):
    global _using_vsscript
    if _using_vsscript:
        global _stored_outputs
        global _environment_id
        if _environment_id is None:
            raise Error('Internal environment id not set. %s called from a filter callback?'%funcname)
            
        return _stored_outputs[_environment_id]
    else:
        return _stored_output
    
def clear_output(int index = 0):
    cdef dict outputs = _get_output_dict("clear_output")
    try:
        del outputs[index]
    except KeyError:
        pass

def clear_outputs():
    cdef dict outputs = _get_output_dict("clear_outputs")
    outputs.clear()

def get_output(int index = 0):
    return _get_output_dict("get_output")[index]

cdef class FuncData(object):
    cdef object func
    cdef Core core
    cdef int id
    
    def __init__(self):
        raise Error('Class cannot be instantiated directly')
        
    def __call__(self, **kwargs):
        return self.func(**kwargs)

cdef FuncData createFuncData(object func, Core core, int id):
    cdef FuncData instance = FuncData.__new__(FuncData)
    instance.func = func
    instance.core = core
    instance.id = id
    return instance
    
cdef class Func(object):
    cdef const VSAPI *funcs
    cdef VSFuncRef *ref
    
    def __init__(self):
        raise Error('Class cannot be instantiated directly')
        
    def __dealloc__(self):
        self.funcs.freeFunc(self.ref)
        
    def __call__(self, **kwargs):
        cdef VSMap *outm
        cdef VSMap *inm
        cdef const VSAPI *vsapi
        cdef const char *error
        vsapi = vpy_getVSApi();
        outm = self.funcs.createMap()
        inm = self.funcs.createMap()
        try:
            dictToMap(kwargs, inm, None, vsapi)
            self.funcs.callFunc(self.ref, inm, outm, NULL, NULL)
            error = self.funcs.getError(outm)
            if error:
                raise Error(error.decode('utf-8'))
            ret = mapToDict(outm, False, False, None, vsapi)
            if not isinstance(ret, dict):
                ret = {'val':ret}
        finally:
            vsapi.freeMap(outm)
            vsapi.freeMap(inm)
        
cdef Func createFuncPython(object func, Core core):
    cdef Func instance = Func.__new__(Func)
    instance.funcs = core.funcs
    if _using_vsscript:
        global _environment_id
        if _environment_id is None:
            raise Error('Internal environment id not set. Report this function wrapper creation error.')
        fdata = createFuncData(func, core, _environment_id)
    else:
        fdata = createFuncData(func, core, 0)
    Py_INCREF(fdata)
    instance.ref = instance.funcs.createFunc(publicFunction, <void *>fdata, freeFunc, core.core, core.funcs)
    return instance
        
cdef Func createFuncRef(VSFuncRef *ref, const VSAPI *funcs):
    cdef Func instance = Func.__new__(Func)
    instance.funcs = funcs
    instance.ref = ref
    return instance

cdef class RawCallbackData(object):
    cdef const VSAPI *funcs
    cdef object callback

    cdef VideoNode node

    cdef object wrap_cb
    cdef object future

    def __init__(self, VideoNode node, object callback = None):
        self.node = node
        self.callback = callback

        self.future = None
        self.wrap_cb = None

    def for_future(self, object future, object wrap_call=None):
        if wrap_call is None:
            wrap_call = lambda func, *args, **kwargs: func(*args, **kwargs)
        self.callback = self.handle_future
        self.future = future
        self.wrap_cb = wrap_call

    def handle_future(self, node, n, result):
        if isinstance(result, Error):
            func = self.future.set_exception
        else:
            func = self.future.set_result

        self.wrap_cb(func, result)

    def receive(self, n, result):
        self.callback(self.node, n, result)


cdef createRawCallbackData(const VSAPI* funcs, VideoNode videonode, object cb, object wrap_call=None):
    cbd = RawCallbackData(videonode, cb)
    if not callable(cb):
        cbd.for_future(cb, wrap_call)
    cbd.funcs = funcs
    return cbd

cdef class CallbackData(object):
    cdef VideoNode node
    cdef const VSAPI *funcs
    cdef object fileobj
    cdef int output
    cdef int requested
    cdef int completed
    cdef int total
    cdef int num_planes
    cdef bint y4m
    cdef dict reorder
    cdef object condition
    cdef object progress_update
    cdef str error

    def __init__(self, fileobj, requested, total, num_planes, y4m, node, progress_update):
        self.fileobj = fileobj
        self.output = 0
        self.requested = requested
        self.completed = 0
        self.total = total
        self.num_planes = num_planes
        self.y4m = y4m
        self.condition = threading.Condition()
        self.node = node
        self.progress_update = progress_update
        self.funcs = (<VideoNode>node).funcs
        self.reorder = {}


cdef class FramePtr(object):
    cdef const VSFrameRef *f
    cdef const VSAPI *funcs

    def __init__(self):
        raise Error('Class cannot be instantiated directly')

    def __dealloc__(self):
        self.funcs.freeFrame(self.f)

cdef FramePtr createFramePtr(const VSFrameRef *f, const VSAPI *funcs):
    cdef FramePtr instance = FramePtr.__new__(FramePtr)    
    instance.f = f
    instance.funcs = funcs
    return instance

cdef void __stdcall frameDoneCallbackRaw(void *data, const VSFrameRef *f, int n, VSNodeRef *node, const char *errormsg) nogil:
    with gil:
        d = <RawCallbackData>data
        if f == NULL:
            result = ''
            if errormsg != NULL:
                result = errormsg.decode('utf-8')
            result = Error(result)

        else:
            result = createConstVideoFrame(f, d.funcs, d.node.core)

        try:
            d.receive(n, result)
        except:
            import traceback
            traceback.print_exc()
        finally:
            Py_DECREF(d)


cdef void __stdcall frameDoneCallbackOutput(void *data, const VSFrameRef *f, int n, VSNodeRef *node, const char *errormsg) with gil:
    cdef VideoFrame frame_obj
    cdef VideoPlane plane
    cdef int x

    cdef CallbackData d = <CallbackData>data
    d.completed += 1

    if f == NULL:
        d.total = d.requested
        if errormsg == NULL:
            d.error = 'Failed to retrieve frame ' + str(n)
        else:
            d.error = 'Failed to retrieve frame ' + str(n) + ' with error: ' + errormsg.decode('utf-8')
        d.output += 1
    else:
        d.reorder[n] = createConstVideoFrame(f, d.funcs, d.node.core)

        while d.output in d.reorder:
            frame_obj = <VideoFrame>d.reorder[d.output]
            try:
                if d.y4m:
                    d.fileobj.write(b'FRAME\n')
                for x in range(frame_obj.format.num_planes):
                    plane = VideoPlane.__new__(VideoPlane, frame_obj, x)
                    d.fileobj.write(plane)
            except:
                d.error = 'File write call returned an error'
                d.total = d.requested

            del d.reorder[d.output]
            d.output += 1

        if d.progress_update is not None:
            try:
                d.progress_update(d.completed, d.total)
            except BaseException as e:
                d.error = 'Progress update caused an exception: ' + str(e)
                d.total = d.requested

    if d.requested < d.total:
        d.node.funcs.getFrameAsync(d.requested, d.node.node, frameDoneCallbackOutput, data)
        d.requested += 1

    d.condition.acquire()
    d.condition.notify()
    d.condition.release()


cdef object mapToDict(const VSMap *map, bint flatten, bint add_cache, Core core, const VSAPI *funcs):
    cdef int numKeys = funcs.propNumKeys(map)
    retdict = {}
    cdef const char *retkey
    cdef char proptype

    for x in range(numKeys):
        retkey = funcs.propGetKey(map, x)
        proptype = funcs.propGetType(map, retkey)

        for y in range(funcs.propNumElements(map, retkey)):
            if proptype == 'i':
                newval = funcs.propGetInt(map, retkey, y, NULL)
            elif proptype == 'f':
                newval = funcs.propGetFloat(map, retkey, y, NULL)
            elif proptype == 's':
                newval = funcs.propGetData(map, retkey, y, NULL)
            elif proptype =='c':
                newval = createVideoNode(funcs.propGetNode(map, retkey, y, NULL), funcs, core)

                if add_cache and not (newval.flags & vapoursynth.nfNoCache):
                    newval = core.std.Cache(clip=newval)

                    if isinstance(newval, dict):
                        newval = newval['dict']
            elif proptype =='v':
                newval = createConstVideoFrame(funcs.propGetFrame(map, retkey, y, NULL), funcs, core)
            elif proptype =='m':
                newval = createFuncRef(funcs.propGetFunc(map, retkey, y, NULL), funcs)

            if y == 0:
                vval = newval
            elif y == 1:
                vval = [vval, newval]
            else:
                vval.append(newval)
        retdict[retkey.decode('utf-8')] = vval

    if not flatten:
        return retdict
    elif len(retdict) == 0:
        return None
    elif len(retdict) == 1:
        a, b = retdict.popitem()
        return b
    else:
        return retdict

cdef void dictToMap(dict ndict, VSMap *inm, Core core, const VSAPI *funcs) except *:
    for key in ndict:
        ckey = key.encode('utf-8')
        val = ndict[key]

        if isinstance(val, (str, bytes, bytearray, VideoNode)):
            val = [val]
        else:
            try:
                iter(val)
            except:
                val = [val]     

        for v in val:
            if isinstance(v, VideoNode):
                if funcs.propSetNode(inm, ckey, (<VideoNode>v).node, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, VideoFrame):
                if funcs.propSetFrame(inm, ckey, (<VideoFrame>v).constf, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, Func):
                if funcs.propSetFunc(inm, ckey, (<Func>v).ref, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif callable(v):
                tf = createFuncPython(v, core)

                if funcs.propSetFunc(inm, ckey, tf.ref, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, int):
                if funcs.propSetInt(inm, ckey, int(v), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, float):
                if funcs.propSetFloat(inm, ckey, float(v), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, str):
                s = str(v).encode('utf-8')

                if funcs.propSetData(inm, ckey, s, -1, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif isinstance(v, (bytes, bytearray)):
                if funcs.propSetData(inm, ckey, v, <int>len(v), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            else:
                raise Error('argument ' + key + ' was passed an unsupported type (' + type(v).__name__ + ')')


cdef void typedDictToMap(dict ndict, dict atypes, VSMap *inm, Core core, const VSAPI *funcs) except *:
    for key in ndict:
        ckey = key.encode('utf-8')
        val = ndict[key]
        if val is None:
            continue

        if isinstance(val, (str, bytes, bytearray, VideoNode)) or not isinstance(val, Iterable):
            val = [val]

        for v in val:
            if atypes[key][:4] == 'clip' and isinstance(v, VideoNode):
                if funcs.propSetNode(inm, ckey, (<VideoNode>v).node, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:5] == 'frame' and isinstance(v, VideoFrame):
                if funcs.propSetFrame(inm, ckey, (<VideoFrame>v).constf, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:4] == 'func' and isinstance(v, Func):
                if funcs.propSetFunc(inm, ckey, (<Func>v).ref, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:4] == 'func' and callable(v):
                tf = createFuncPython(v, core)
                if funcs.propSetFunc(inm, ckey, tf.ref, 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:3] == 'int':
                if funcs.propSetInt(inm, ckey, int(v), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:5] == 'float':
                if funcs.propSetFloat(inm, ckey, float(v), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            elif atypes[key][:4] == 'data':
                if not isinstance(v, (str, bytes, bytearray)):
                    v = str(v)
                if isinstance(v, str):
                    s = v.encode('utf-8')
                else:
                    s = v
                if funcs.propSetData(inm, ckey, s, <int>len(s), 1) != 0:
                    raise Error('not all values are of the same type in ' + key)
            else:
                raise Error('argument ' + key + ' was passed an unsupported type (expected ' + atypes[key] + ' compatible type but got ' + type(v).__name__ + ')')
        if len(val) == 0:
        # set an empty key if it's an empty array
            if atypes[key][:4] == 'clip':
                funcs.propSetNode(inm, ckey, NULL, 2)
            elif atypes[key][:5] == 'frame':
                funcs.propSetFrame(inm, ckey, NULL, 2)
            elif atypes[key][:4] == 'func':
                funcs.propSetFunc(inm, ckey, NULL, 2)
            elif atypes[key][:3] == 'int':
                funcs.propSetInt(inm, ckey, 0, 2)
            elif atypes[key][:5] == 'float':
                funcs.propSetFloat(inm, ckey, 0, 2)
            elif atypes[key][:4] == 'data':
                funcs.propSetData(inm, ckey, NULL, 0, 2)
            else:
                raise Error('argument ' + key + ' has an unknown type: ' + atypes[key])

cdef class Format(object):
    cdef readonly int id
    cdef readonly str name
    cdef readonly object color_family
    cdef readonly object sample_type
    cdef readonly int bits_per_sample
    cdef readonly int bytes_per_sample
    cdef readonly int subsampling_w
    cdef readonly int subsampling_h
    cdef readonly int num_planes

    def __init__(self):
        raise Error('Class cannot be instantiated directly')

    def _as_dict(self):
        return {
            'color_family': self.color_family,
            'sample_type': self.sample_type,
            'bits_per_sample': self.bits_per_sample,
            'subsampling_w': self.subsampling_w,
            'subsampling_h': self.subsampling_h
        }

    def replace(self, **kwargs):
        core = kwargs.pop("core", None) or get_core()
        vals = self._as_dict()
        vals.update(**kwargs)
        return core.register_format(**vals)

    def __int__(self):
        return self.id

    def __str__(self):
        return ('Format Descriptor\n'
               f'\tId: {self.id:d}\n'
               f'\tName: {self.name}\n'
               f'\tColor Family: {self.color_family.name}\n'
{'fraction_non_alphanumeric': 0.09522390283883467, 'fraction_numerical': 0.005464073715321759, 'mean_word_length': 2.112313519363067, 'pattern_counts': {'":': 14, '<': 33, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1045659', 'n_tokens_mistral': 12567, 'n_tokens_neox': 12029, 'n_words': 3641}