<commit_before>/*
* Copyright (c) 2012-2020 <NAME>
*
* This file is part of VapourSynth.
*
* VapourSynth is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* VapourSynth is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with VapourSynth; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "vscore.h"
#include "VSHelper4.h"
#include "version.h"
#include "cpufeatures.h"
#ifndef VS_TARGET_OS_WINDOWS
#include <dirent.h>
#include <cstddef>
#include <unistd.h>
#include "settings.h"
#endif
#include <cassert>
#include <queue>
#include <bitset>

#ifdef VS_TARGET_CPU_X86
#include "x86utils.h"
#endif

#ifdef VS_TARGET_OS_WINDOWS
#include <shlobj.h>
#include <locale>
#include "../common/vsutf16.h"
#endif

// Internal filter headers
#include "internalfilters.h"

#ifdef VS_USE_MIMALLOC
#   include <mimalloc-new-delete.h>
#endif

using namespace vsh;

static inline bool isAlpha(char c) {
    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}

static inline bool isAlphaNumUnderscore(char c) {
    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
}

static bool isValidIdentifier(const std::string &s) {
    size_t len = s.length();
    if (!len)
        return false;

    if (!isAlpha(s[0]))
        return false;
    for (size_t i = 1; i < len; i++)
        if (!isAlphaNumUnderscore(s[i]))
            return false;
    return true;
}

#ifdef VS_TARGET_OS_WINDOWS
static std::wstring readRegistryValue(const wchar_t *keyName, const wchar_t *valueName) {
    HKEY hKey;
    LONG lRes = RegOpenKeyEx(HKEY_CURRENT_USER, keyName, 0, KEY_READ, &hKey);
    if (lRes != ERROR_SUCCESS) {
        lRes = RegOpenKeyEx(HKEY_LOCAL_MACHINE, keyName, 0, KEY_READ, &hKey);
        if (lRes != ERROR_SUCCESS)
            return std::wstring();
    }
    WCHAR szBuffer[512];
    DWORD dwBufferSize = sizeof(szBuffer);
    ULONG nError;
    nError = RegQueryValueEx(hKey, valueName, 0, nullptr, (LPBYTE)szBuffer, &dwBufferSize);
    RegCloseKey(hKey);
    if (ERROR_SUCCESS == nError)
        return szBuffer;
    return std::wstring();
}
#endif

VSFrameContext::VSFrameContext(NodeOutputKey key, const PVSFrameContext &notify) :
    refcount(1), reqOrder(notify->reqOrder), external(false), lockOnOutput(true), frameDone(nullptr),  userData(nullptr), key(key), frameContext() {
    notifyCtxList.push_back(notify);
}

VSFrameContext::VSFrameContext(int n, VSNode *node, VSFrameDoneCallback frameDone, void *userData, bool lockOnOutput) :
    refcount(1), reqOrder(0), external(true), lockOnOutput(lockOnOutput), frameDone(frameDone), userData(userData), key(node, n), frameContext() {
}

bool VSFrameContext::setError(const std::string &errorMsg) {
    bool prevState = error;
    error = true;
    if (!prevState)
        errorMessage = errorMsg;
    return prevState;
}

///////////////

bool VSMap::isV3Compatible() const noexcept {
    for (const auto &iter : data->data) {
        if (iter.second->type() == ptAudioNode || iter.second->type() == ptAudioFrame || iter.second->type() == ptUnset)
            return false;
    }
    return true;
}

VSFunction::VSFunction(VSPublicFunction func, void *userData, VSFreeFunctionData freeFunction, VSCore *core, int apiMajor) : refcount(1), func(func), userData(userData), freeFunction(freeFunction), core(core), apiMajor(apiMajor) {
    core->functionInstanceCreated();
}

VSFunction::~VSFunction() {
    if (freeFunction)
        freeFunction(userData);
    core->functionInstanceDestroyed();
}

void VSFunction::call(const VSMap *in, VSMap *out) {
    if (apiMajor == VAPOURSYNTH3_API_MAJOR && !in->isV3Compatible()) {
        vs_internal_vsapi.mapSetError(out, "Function was passed values that are unknown to its API version");
        return;
    }
        
    func(in, out, userData, core, getVSAPIInternal(apiMajor));
}

void MemoryUse::add(size_t bytes) {
    used.fetch_add(bytes, std::memory_order_relaxed);
}

void MemoryUse::subtract(size_t bytes) {
    size_t tmp = used.fetch_sub(bytes, std::memory_order_relaxed) - bytes;
    if (freeOnZero && !tmp)
        delete this;
}

size_t MemoryUse::memoryUse() {
    return used;
}

size_t MemoryUse::getLimit() {
    return maxMemoryUse.load(std::memory_order_relaxed);
}

int64_t MemoryUse::setMaxMemoryUse(int64_t bytes) {
    if (bytes > 0 && static_cast<uint64_t>(bytes) <= SIZE_MAX)
        maxMemoryUse.store(static_cast<size_t>(bytes), std::memory_order_seq_cst);
    return maxMemoryUse;
}

bool MemoryUse::isOverLimit() {
    return used.load(std::memory_order_relaxed) > maxMemoryUse.load(std::memory_order_relaxed);
}

void MemoryUse::signalFree() {
    freeOnZero = true;
    if (!used)
        delete this;
}

MemoryUse::MemoryUse() : used(0), freeOnZero(false) {
    // 1GB
    setMaxMemoryUse(1024 * 1024 * 1024);

    // set 4GB as default on systems with (probably) 64bit address space
    if (sizeof(void *) >= 8)
        setMaxMemoryUse(static_cast<int64_t>(4) * 1024 * 1024 * 1024);
}

///////////////

VSPlaneData::VSPlaneData(size_t dataSize, MemoryUse &mem) noexcept : refcount(1), mem(mem), size(dataSize + 2 * VSFrame::guardSpace) {
    data = vsh_aligned_malloc<uint8_t>(size + 2 * VSFrame::guardSpace, VSFrame::alignment);
    assert(data);
    if (!data)
        VS_FATAL_ERROR("Failed to allocate memory for plane. Out of memory.");

    mem.add(size);
#ifdef VS_FRAME_GUARD
    for (size_t i = 0; i < VSFrame::guardSpace / sizeof(VS_FRAME_GUARD_PATTERN); i++) {
        reinterpret_cast<uint32_t *>(data)[i] = VS_FRAME_GUARD_PATTERN;
        reinterpret_cast<uint32_t *>(data + size - VSFrame::guardSpace)[i] = VS_FRAME_GUARD_PATTERN;
    }
#endif
}

VSPlaneData::VSPlaneData(const VSPlaneData &d) noexcept : refcount(1), mem(d.mem), size(d.size) {
    data = vsh_aligned_malloc<uint8_t>(size, VSFrame::alignment);
    assert(data);
    if (!data)
        VS_FATAL_ERROR("Failed to allocate memory for plane in copy constructor. Out of memory.");

    mem.add(size);
    memcpy(data, d.data, size);
}

VSPlaneData::~VSPlaneData() {
    vsh_aligned_free(data);
    mem.subtract(size);
}

bool VSPlaneData::unique() noexcept {
    return (refcount == 1);
}

void VSPlaneData::add_ref() noexcept {
    ++refcount;
}

void VSPlaneData::release() noexcept {
    if (!--refcount)
        delete this;
}

///////////////

VSFrame::VSFrame(const VSVideoFormat &f, int width, int height, const VSFrame *propSrc, VSCore *core) noexcept : refcount(1), contentType(mtVideo), v3format(nullptr), width(width), height(height), properties(propSrc ? &propSrc->properties : nullptr), core(core) {
    if (width <= 0 || height <= 0)
        core->logFatal("Error in frame creation: dimensions are negative (" + std::to_string(width) + "x" + std::to_string(height) + ")");

    format.vf = f;
    numPlanes = format.vf.numPlanes;

    stride[0] = (width * (f.bytesPerSample) + (alignment - 1)) & ~(alignment - 1);

    if (numPlanes == 3) {
        int plane23 = ((width >> format.vf.subSamplingW) * (format.vf.bytesPerSample) + (alignment - 1)) & ~(alignment - 1);
        stride[1] = plane23;
        stride[2] = plane23;
    } else {
        stride[1] = 0;
        stride[2] = 0;
    }

    data[0] = new VSPlaneData(stride[0] * height, *core->memory);
    if (numPlanes == 3) {
        size_t size23 = stride[1] * (height >> format.vf.subSamplingH);
        data[1] = new VSPlaneData(size23, *core->memory);
        data[2] = new VSPlaneData(size23, *core->memory);
    }
}

VSFrame::VSFrame(const VSVideoFormat &f, int width, int height, const VSFrame * const *planeSrc, const int *plane, const VSFrame *propSrc, VSCore *core) noexcept : refcount(1), contentType(mtVideo), v3format(nullptr), width(width), height(height), properties(propSrc ? &propSrc->properties : nullptr), core(core) {
    if (width <= 0 || height <= 0)
        core->logFatal("Error in frame creation: dimensions are negative " + std::to_string(width) + "x" + std::to_string(height));

    format.vf = f;
    numPlanes = format.vf.numPlanes;

    stride[0] = (width * (format.vf.bytesPerSample) + (alignment - 1)) & ~(alignment - 1);

    if (numPlanes == 3) {
        int plane23 = ((width >> format.vf.subSamplingW) * (format.vf.bytesPerSample) + (alignment - 1)) & ~(alignment - 1);
        stride[1] = plane23;
        stride[2] = plane23;
    } else {
        stride[1] = 0;
        stride[2] = 0;
    }

    for (int i = 0; i < numPlanes; i++) {
        if (planeSrc[i]) {
            if (plane[i] < 0 || plane[i] >= planeSrc[i]->format.vf.numPlanes)
                core->logFatal("Error in frame creation: plane " + std::to_string(plane[i]) + " does not exist in the source frame");
            if (planeSrc[i]->getHeight(plane[i]) != getHeight(i) || planeSrc[i]->getWidth(plane[i]) != getWidth(i))
                core->logFatal("Error in frame creation: dimensions of plane " + std::to_string(plane[i]) + " do not match. Source: " + std::to_string(planeSrc[i]->getWidth(plane[i])) + "x" + std::to_string(planeSrc[i]->getHeight(plane[i])) + "; destination: " + std::to_string(getWidth(i)) + "x" + std::to_string(getHeight(i)));
            data[i] = planeSrc[i]->data[plane[i]];
            data[i]->add_ref();
        } else {
            if (i == 0) {
                data[i] = new VSPlaneData(stride[i] * height, *core->memory);
            } else {
                data[i] = new VSPlaneData(stride[i] * (height >> format.vf.subSamplingH), *core->memory);
            }
        }
    }
}

VSFrame::VSFrame(const VSAudioFormat &f, int numSamples, const VSFrame *propSrc, VSCore *core) noexcept
    : refcount(1), contentType(mtAudio), v3format(nullptr), properties(propSrc ? &propSrc->properties : nullptr), core(core) {
    if (numSamples <= 0)
        core->logFatal("Error in frame creation: bad number of samples (" + std::to_string(numSamples) + ")");
    
    format.af = f;
    numPlanes = format.af.numChannels;

    width = numSamples;

    stride[0] = format.af.bytesPerSample * VS_AUDIO_FRAME_SAMPLES;

    data[0] = new VSPlaneData(stride[0] * format.af.numChannels, *core->memory);
}

VSFrame::VSFrame(const VSAudioFormat &f, int numSamples, const VSFrame * const *channelSrc, const int *channel, const VSFrame *propSrc, VSCore *core) noexcept
    : refcount(1), contentType(mtAudio), v3format(nullptr), properties(propSrc ? &propSrc->properties : nullptr), core(core) {
    if (numSamples <= 0)
        core->logFatal("Error in frame creation: bad number of samples (" + std::to_string(numSamples) + ")");

    format.af = f;
    numPlanes = format.af.numChannels;

    width = numSamples;

    stride[0] = format.af.bytesPerSample * VS_AUDIO_FRAME_SAMPLES;

    data[0] = new VSPlaneData(stride[0] * format.af.numChannels, *core->memory);

    for (int i = 0; i < numPlanes; i++) {
        if (channelSrc[i]) {
            if (channel[i] < 0 || channel[i] >= channelSrc[i]->format.af.numChannels)
                core->logFatal("Error in frame creation: channel " + std::to_string(channel[i]) + " does not exist in the source frame");
            if (channelSrc[i]->getFrameLength() != getFrameLength())
                core->logFatal("Error in frame creation: length of frame does not match. Source: " + std::to_string(channelSrc[i]->getFrameLength()) + "; destination: " + std::to_string(getFrameLength()));
            memcpy(getWritePtr(i), channelSrc[i]->getReadPtr(channel[i]), getFrameLength() * format.af.bytesPerSample);
        }
    }
}

VSFrame::VSFrame(const VSFrame &f) noexcept : refcount(1), v3format(nullptr) {
    contentType = f.contentType;
    data[0] = f.data[0];
    data[1] = f.data[1];
    data[2] = f.data[2];
    data[0]->add_ref();
    if (data[1]) {
        data[1]->add_ref();
        data[2]->add_ref();
    }
    format = f.format;
    numPlanes = f.numPlanes;
    width = f.width;
    height = f.height;
    stride[0] = f.stride[0];
    stride[1] = f.stride[1];
    stride[2] = f.stride[2];
    properties = f.properties;
    core = f.core;
}

VSFrame::~VSFrame() {
    data[0]->release();
    if (data[1]) {
        data[1]->release();
        data[2]->release();
    }
}

const vs3::VSVideoFormat *VSFrame::getVideoFormatV3() const noexcept {
    assert(contentType == mtVideo);
    if (!v3format)
        v3format = core->VideoFormatToV3(format.vf);
    return v3format;
}

ptrdiff_t VSFrame::getStride(int plane) const {
    assert(contentType == mtVideo);
    if (plane < 0 || plane >= numPlanes)
        return 0;
    return stride[plane];
}

const uint8_t *VSFrame::getReadPtr(int plane) const {
    if (plane < 0 || plane >= numPlanes)
        return nullptr;

    if (contentType == mtVideo)
        return data[plane]->data + guardSpace;
    else
        return data[0]->data + guardSpace + plane * stride[0];
}

uint8_t *VSFrame::getWritePtr(int plane) {
    if (plane < 0 || plane >= numPlanes)
        return nullptr;

    // copy the plane data if this isn't the only reference
    if (contentType == mtVideo) {
        if (!data[plane]->unique()) {
            VSPlaneData *old = data[plane];
            data[plane] = new VSPlaneData(*data[plane]);
            old->release();
        }

        return data[plane]->data + guardSpace;
    } else {
        if (!data[0]->unique()) {
            VSPlaneData *old = data[0];
            data[0] = new VSPlaneData(*data[0]);
            old->release();
        }

        return data[0]->data + guardSpace + plane * stride[0];
    }
}

#ifdef VS_FRAME_GUARD
bool VSFrame::verifyGuardPattern() const {
    for (int p = 0; p < ((contentType == mtVideo) ? numPlanes : 1); p++) {
        for (size_t i = 0; i < guardSpace / sizeof(VS_FRAME_GUARD_PATTERN); i++) {
            uint32_t p1 = reinterpret_cast<uint32_t *>(data[p]->data)[i];
            uint32_t p2 = reinterpret_cast<uint32_t *>(data[p]->data + data[p]->size - guardSpace)[i];
            if (reinterpret_cast<uint32_t *>(data[p]->data)[i] != VS_FRAME_GUARD_PATTERN ||
                reinterpret_cast<uint32_t *>(data[p]->data + data[p]->size - guardSpace)[i] != VS_FRAME_GUARD_PATTERN)
                return false;
        }
    }

    return true;
}
#endif

struct split1 {
    enum empties_t { empties_ok, no_empties };
};

template <typename Container>
Container& split(
    Container& result,
    const typename Container::value_type& s,
    const typename Container::value_type& delimiters,
    split1::empties_t empties = split1::empties_ok) {
    result.clear();
    size_t current;
    size_t next = -1;
    do {
        if (empties == split1::no_empties) {
            next = s.find_first_not_of(delimiters, next + 1);
            if (next == Container::value_type::npos) break;
            next -= 1;
        }
        current = next + 1;
        next = s.find_first_of(delimiters, current);
        result.push_back(s.substr(current, next - current));
    } while (next != Container::value_type::npos);
    return result;
}

void VSPluginFunction::parseArgString(const std::string &argString, std::vector<FilterArgument> &argsOut, int apiMajor) {
    std::vector<std::string> argList;
    split(argList, argString, std::string(";"), split1::no_empties);

    argsOut.reserve(argList.size());
    for (const std::string &arg : argList) {
        std::vector<std::string> argParts;
        split(argParts, arg, std::string(":"), split1::no_empties);

        if (argParts.size() == 1 && argParts[0] == "any") {
            argsOut.emplace_back("", ptUnset, false, false, false);
            continue;
        }

        if (argParts.size() < 2)
            throw std::runtime_error("Invalid argument specifier '" + arg + "'. It appears to be incomplete.");

        bool arr = false;
        bool opt = false;
        bool empty = false;

        VSPropertyType type = ptUnset;
        const std::string &argName = argParts[0];
        std::string &typeName = argParts[1];

        if (typeName.length() > 2 && typeName.substr(typeName.length() - 2) == "[]") {
            typeName.resize(typeName.length() - 2);
            arr = true;
        }

        if (typeName == "int") {
            type = ptInt;
        } else if (typeName == "float") {
            type = ptFloat;
        } else if (typeName == "data") {
            type = ptData;
        } else if ((typeName == "vnode" && apiMajor > VAPOURSYNTH3_API_MAJOR) || (apiMajor == VAPOURSYNTH3_API_MAJOR && typeName == "clip")) {
            type = ptVideoNode;
        } else if (typeName == "anode" && apiMajor > VAPOURSYNTH3_API_MAJOR) {
            type = ptAudioNode;
        } else if ((typeName == "vframe" && apiMajor > VAPOURSYNTH3_API_MAJOR) || (apiMajor == VAPOURSYNTH3_API_MAJOR && typeName == "frame")) {
            type = ptVideoFrame;
        } else if (typeName == "aframe" && apiMajor > VAPOURSYNTH3_API_MAJOR) {
            type = ptAudioFrame;
        } else if (typeName == "func") {
            type = ptFunction;
        } else {
            throw std::runtime_error("Argument '" + argName + "' has invalid type '" + typeName + "'.");
        }

        for (size_t i = 2; i < argParts.size(); i++) {
            if (argParts[i] == "opt") {
                if (opt)
                    throw std::runtime_error("Argument '" + argName + "' has duplicate argument specifier '" + argParts[i] + "'");
                opt = true;
            } else if (argParts[i] == "empty") {
                if (empty)
                    throw std::runtime_error("Argument '" + argName + "' has duplicate argument specifier '" + argParts[i] + "'");
                empty = true;
            } else {
                throw std::runtime_error("Argument '" + argName + "' has unknown argument modifier '" + argParts[i] + "'");
            }
        }

        if (!isValidIdentifier(argName))
            throw std::runtime_error("Argument name '" + argName + "' contains illegal characters.");

        if (empty && !arr)
            throw std::runtime_error("Argument '" + argName + "' is not an array. Only array arguments can have the empty flag set.");

        argsOut.emplace_back(argName, type, arr, empty, opt);
    }
}

VSPluginFunction::VSPluginFunction(const std::string &name, const std::string &argString, const std::string &returnType, VSPublicFunction func, void *functionData, VSPlugin *plugin)
    : func(func), functionData(functionData), plugin(plugin), name(name), argString(argString), returnType(returnType) {
    parseArgString(argString, inArgs, plugin->apiMajor);
    if (plugin->apiMajor == 3)
        this->argString = getV4ArgString(); // construct to V4 equivalent arg string
    if (returnType != "any")
        parseArgString(returnType, retArgs, plugin->apiMajor);
}

VSMap *VSPluginFunction::invoke(const VSMap &args) {
    VSMap *v = new VSMap;

    try {
        std::set<std::string> remainingArgs;
        for (size_t i = 0; i < args.size(); i++)
            remainingArgs.insert(args.key(i));

        for (const FilterArgument &fa : inArgs) {
            // ptUnset as an argument type means any value is accepted beyond the declared ones
            if (fa.type == ptUnset) {
                // this will always be the last argument, therefore it's safe to clear out the remaining ones since they'll
                // already have been checked
                remainingArgs.clear();
                continue;
            }

            int propType = vs_internal_vsapi.mapGetType(&args, fa.name.c_str());

            if (propType != ptUnset) {
                remainingArgs.erase(fa.name);

                if (fa.type != propType)
                    throw VSException(name + ": argument " + fa.name + " is not of the correct type");

                VSArrayBase *arr = args.find(fa.name);

                if (!fa.arr && arr->size() > 1)
                    throw VSException(name + ": argument " + fa.name + " is not of array type but more than one value was supplied");

                if (!fa.empty && arr->size() < 1)
                    throw VSException(name + ": argument " + fa.name + " does not accept empty arrays");

            } else if (!fa.opt) {
                throw VSException(name + ": argument " + fa.name + " is required");
            }
        }

        if (!remainingArgs.empty()) {
            auto iter = remainingArgs.cbegin();
            std::string s = *iter;
            ++iter;
            for (; iter != remainingArgs.cend(); ++iter)
                s += ", " + *iter;
            throw VSException(name + ": no argument(s) named " + s);
        }

        bool enableGraphInspection = plugin->core->enableGraphInspection;
        if (enableGraphInspection) {
            plugin->core->functionFrame = std::make_shared<VSFunctionFrame>(name, new VSMap(&args), plugin->core->functionFrame);
        }
        func(&args, v, functionData, plugin->core, getVSAPIInternal(plugin->apiMajor));
        if (enableGraphInspection) {
            assert(plugin->core->functionFrame);
            plugin->core->functionFrame = plugin->core->functionFrame->next;
        }

        if (plugin->apiMajor == VAPOURSYNTH3_API_MAJOR && !args.isV3Compatible())
            plugin->core->logFatal(name + ": filter node returned not yet supported type");

    } catch (VSException &e) {
        vs_internal_vsapi.mapSetError(v, e.what());
    }

    return v;
}

bool VSPluginFunction::isV3Compatible() const {
    for (const auto &iter : inArgs)
        if (iter.type == ptAudioNode || iter.type == ptAudioFrame)
            return false;
    for (const auto &iter : retArgs)
        if (iter.type == ptAudioNode || iter.type == ptAudioFrame)
            return false;
    return true;
}

std::string VSPluginFunction::getV4ArgString() const {
    std::string tmp;
    for (const auto &iter : inArgs) {
        assert(iter.type != ptAudioNode && iter.type != ptAudioFrame);

        tmp += iter.name + ":";

        switch (iter.type) {
            case ptInt:
                tmp += "int"; break;
            case ptFloat:
                tmp += "float"; break;
            case ptData:
                tmp += "data"; break;
            case ptVideoNode:
                tmp += "vnode"; break;
            case ptVideoFrame:
                tmp += "vframe"; break;
            case ptFunction:
                tmp += "func"; break;
            default:
                assert(false);
        }
        if (iter.arr)
            tmp += "[]";
        if (iter.opt)
            tmp += ":opt";
        if (iter.empty)
            tmp += ":empty";
        tmp += ";";
    }
    return tmp;
}

std::string VSPluginFunction::getV3ArgString() const {
    std::string tmp;
    for (const auto &iter : inArgs) {
        assert(iter.type != ptAudioNode && iter.type != ptAudioFrame);

        tmp += iter.name + ":";

        switch (iter.type) {
            case ptInt:
                tmp += "int"; break;
            case ptFloat:
                tmp += "float"; break;
            case ptData:
                tmp += "data"; break;
            case ptVideoNode:
                tmp += "clip"; break;
            case ptVideoFrame:
                tmp += "frame"; break;
            case ptFunction:
                tmp += "func"; break;
            default:
                assert(false);
        }
        if (iter.arr)
            tmp += "[]";
        if (iter.opt)
            tmp += ":opt";
        if (iter.empty)
            tmp += ":empty";
        tmp += ";";
    }
    return tmp;
}

const std::string &VSPluginFunction::getName() const {
    return name;
}

const std::string &VSPluginFunction::getArguments() const {
    return argString;
}

const std::string &VSPluginFunction::getReturnType() const {
    return returnType;
}

struct MakeLinearWrapper {
    vs3::VSFilterGetFrame getFrameFunc;
    VSFilterFree freeFunc;
    void *instanceData;
    int threshold;
    int lastFrame = -1;

    MakeLinearWrapper(VSFilterGetFrame g, VSFilterFree f, void *instanceData, int threshold) :
        getFrameFunc(reinterpret_cast<vs3::VSFilterGetFrame>(g)),
        freeFunc(f), instanceData(instanceData), threshold(threshold) {}

    static const VSFrame *VS_CC getFrame(int n, int activationReason, void *instanceData, void **frameData, VSFrameContext *frameCtx, VSCore *core, const VSAPI *vsapi);
    static void VS_CC freeFilter(void *instanceData, VSCore *core, const VSAPI *vsapi);
};

VSNode::VSNode(const VSMap *in, VSMap *out, const std::string &name, vs3::VSFilterInit init, VSFilterGetFrame getFrame, VSFilterFree freeFunc, VSFilterMode filterMode, int flags, void *instanceData, int apiMajor, VSCore *core) :
    refcount(1), nodeType(mtVideo), instanceData(instanceData), name(name), filterGetFrame(getFrame), freeFunc(freeFunc), filterMode(filterMode), apiMajor(apiMajor), core(core), serialFrame(-1) {

    if (flags & ~(vs3::nfNoCache | vs3::nfIsCache | vs3::nfMakeLinear))
        throw VSException("Filter " + name  + " specified unknown flags");

    if ((flags & vs3::nfIsCache) && !(flags & vs3::nfNoCache))
        throw VSException("Filter " + name + " specified an illegal combination of flags (nfNoCache must always be set with nfIsCache)");


    VSMap inval(in);
    init(&inval, out, &this->instanceData, this, core, reinterpret_cast<const vs3::VSAPI3 *>(getVSAPIInternal(3)));

    if (out->hasError()) {
        throw VSException(vs_internal_vsapi.mapGetError(out));
    }

    if (vi.format.colorFamily == 0) {
        throw VSException("Filter " + name + " didn't set videoinfo");
    }

    if (vi.numFrames <= 0) {
        throw VSException("Filter " + name + " returned zero or negative frame count");
    }

    core->filterInstanceCreated();

    // Scan the in map for clips, these are probably the real dependencies
    // Worst case there are false positives and an extra cache gets activated
    // NoCache is generally the equivalent strict spatial for filters

    int requestPattern = !!(flags & vs3::nfNoCache) ? rpNoFrameReuse : rpGeneral;
    int numKeys = vs_internal_vsapi.mapNumKeys(in);

    bool makeLinear = (flags & vs3::nfMakeLinear);
    bool hasVideoNodes = false;
    for (int i = 0; i < numKeys; i++) {
        const char *key = vs_internal_vsapi.mapGetKey(in, i);
        if (vs_internal_vsapi.mapGetType(in, key) == ptVideoNode) {
            int numElems = vs_internal_vsapi.mapNumElements(in, key);
            for (int j = 0; j < numElems; j++) {
                hasVideoNodes = true;
                VSNode *sn = vs_internal_vsapi.mapGetNode(in, key, j, nullptr);
                this->dependencies.push_back({sn, requestPattern});
                sn->addConsumer(this, requestPattern);
            }
        }
    }

    if (makeLinear && !hasVideoNodes) {
        this->apiMajor = 4;
        MakeLinearWrapper *wrapper = new MakeLinearWrapper(filterGetFrame, freeFunc, instanceData, setLinear());
        filterGetFrame = MakeLinearWrapper::getFrame;
        this->freeFunc = MakeLinearWrapper::freeFilter;
        this->instanceData = reinterpret_cast<void *>(wrapper);
    }

    if (core->enableGraphInspection) {
        functionFrame = core->functionFrame;
    }
}

VSNode::VSNode(const std::string &name, const VSVideoInfo *vi, VSFilterGetFrame getFrame, VSFilterFree freeFunc, VSFilterMode filterMode, const VSFilterDependency *dependencies, int numDeps, void *instanceData, int apiMajor, VSCore *core) :
    refcount(1), nodeType(mtVideo), instanceData(instanceData), name(name), filterGetFrame(getFrame), freeFunc(freeFunc), filterMode(filterMode), apiMajor(apiMajor), core(core), serialFrame(-1) {

    if (!core->isValidVideoInfo(*vi))
        throw VSException("The VSVideoInfo structure passed by " + name + " is invalid.");

    this->vi = *vi;
    this->v3vi = core->VideoInfoToV3(*vi);

    core->filterInstanceCreated();

    this->dependencies.reserve(numDeps);
    for (int i = 0; i < numDeps; i++) {
        this->dependencies.push_back(dependencies[i]);
        dependencies[i].source->add_ref();
        dependencies[i].source->addConsumer(this, dependencies[i].requestPattern);
    }

    if (core->enableGraphInspection) {
        functionFrame = core->functionFrame;
    }
}

VSNode::VSNode(const std::string &name, const VSAudioInfo *ai, VSFilterGetFrame getFrame, VSFilterFree freeFunc, VSFilterMode filterMode, const VSFilterDependency *dependencies, int numDeps, void *instanceData, int apiMajor, VSCore *core) :
    refcount(1), nodeType(mtAudio), instanceData(instanceData), name(name), filterGetFrame(getFrame), freeFunc(freeFunc), filterMode(filterMode), apiMajor(apiMajor), core(core), serialFrame(-1) {

    if (!core->isValidAudioInfo(*ai))<commit_msg>Initialize processing time variable<commit_after>/*
* Copyright (c) 2012-2020 <NAME>
*
* This file is part of VapourSynth.
*
* VapourSynth is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* VapourSynth is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with VapourSynth; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/

#include "vscore.h"
#include "VSHelper4.h"
#include "version.h"
#include "cpufeatures.h"
#ifndef VS_TARGET_OS_WINDOWS
#include <dirent.h>
#include <cstddef>
#include <unistd.h>
#include "settings.h"
#endif
#include <cassert>
#include <queue>
#include <bitset>

#ifdef VS_TARGET_CPU_X86
#include "x86utils.h"
#endif

#ifdef VS_TARGET_OS_WINDOWS
#include <shlobj.h>
#include <locale>
#include "../common/vsutf16.h"
#endif

// Internal filter headers
#include "internalfilters.h"

#ifdef VS_USE_MIMALLOC
#   include <mimalloc-new-delete.h>
#endif

using namespace vsh;

static inline bool isAlpha(char c) {
    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');
}

static inline bool isAlphaNumUnderscore(char c) {
    return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_';
}

static bool isValidIdentifier(const std::string &s) {
    size_t len = s.length();
    if (!len)
        return false;

    if (!isAlpha(s[0]))
        return false;
    for (size_t i = 1; i < len; i++)
        if (!isAlphaNumUnderscore(s[i]))
            return false;
    return true;
}

#ifdef VS_TARGET_OS_WINDOWS
static std::wstring readRegistryValue(const wchar_t *keyName, const wchar_t *valueName) {
    HKEY hKey;
    LONG lRes = RegOpenKeyEx(HKEY_CURRENT_USER, keyName, 0, KEY_READ, &hKey);
    if (lRes != ERROR_SUCCESS) {
        lRes = RegOpenKeyEx(HKEY_LOCAL_MACHINE, keyName, 0, KEY_READ, &hKey);
        if (lRes != ERROR_SUCCESS)
            return std::wstring();
    }
    WCHAR szBuffer[512];
    DWORD dwBufferSize = sizeof(szBuffer);
    ULONG nError;
    nError = RegQueryValueEx(hKey, valueName, 0, nullptr, (LPBYTE)szBuffer, &dwBufferSize);
    RegCloseKey(hKey);
    if (ERROR_SUCCESS == nError)
        return szBuffer;
    return std::wstring();
}
#endif

VSFrameContext::VSFrameContext(NodeOutputKey key, const PVSFrameContext &notify) :
    refcount(1), reqOrder(notify->reqOrder), external(false), lockOnOutput(true), frameDone(nullptr),  userData(nullptr), key(key), frameContext() {
    notifyCtxList.push_back(notify);
}

VSFrameContext::VSFrameContext(int n, VSNode *node, VSFrameDoneCallback frameDone, void *userData, bool lockOnOutput) :
    refcount(1), reqOrder(0), external(true), lockOnOutput(lockOnOutput), frameDone(frameDone), userData(userData), key(node, n), frameContext() {
}

bool VSFrameContext::setError(const std::string &errorMsg) {
    bool prevState = error;
    error = true;
    if (!prevState)
        errorMessage = errorMsg;
    return prevState;
}

///////////////

bool VSMap::isV3Compatible() const noexcept {
    for (const auto &iter : data->data) {
        if (iter.second->type() == ptAudioNode || iter.second->type() == ptAudioFrame || iter.second->type() == ptUnset)
            return false;
    }
    return true;
}

VSFunction::VSFunction(VSPublicFunction func, void *userData, VSFreeFunctionData freeFunction, VSCore *core, int apiMajor) : refcount(1), func(func), userData(userData), freeFunction(freeFunction), core(core), apiMajor(apiMajor) {
    core->functionInstanceCreated();
}

VSFunction::~VSFunction() {
    if (freeFunction)
        freeFunction(userData);
    core->functionInstanceDestroyed();
}

void VSFunction::call(const VSMap *in, VSMap *out) {
    if (apiMajor == VAPOURSYNTH3_API_MAJOR && !in->isV3Compatible()) {
        vs_internal_vsapi.mapSetError(out, "Function was passed values that are unknown to its API version");
        return;
    }
        
    func(in, out, userData, core, getVSAPIInternal(apiMajor));
}

void MemoryUse::add(size_t bytes) {
    used.fetch_add(bytes, std::memory_order_relaxed);
}

void MemoryUse::subtract(size_t bytes) {
    size_t tmp = used.fetch_sub(bytes, std::memory_order_relaxed) - bytes;
    if (freeOnZero && !tmp)
        delete this;
}

size_t MemoryUse::memoryUse() {
    return used;
}

size_t MemoryUse::getLimit() {
    return maxMemoryUse.load(std::memory_order_relaxed);
}

int64_t MemoryUse::setMaxMemoryUse(int64_t bytes) {
    if (bytes > 0 && static_cast<uint64_t>(bytes) <= SIZE_MAX)
        maxMemoryUse.store(static_cast<size_t>(bytes), std::memory_order_seq_cst);
    return maxMemoryUse;
}

bool MemoryUse::isOverLimit() {
    return used.load(std::memory_order_relaxed) > maxMemoryUse.load(std::memory_order_relaxed);
}

void MemoryUse::signalFree() {
    freeOnZero = true;
    if (!used)
        delete this;
}

MemoryUse::MemoryUse() : used(0), freeOnZero(false) {
    // 1GB
    setMaxMemoryUse(1024 * 1024 * 1024);

    // set 4GB as default on systems with (probably) 64bit address space
    if (sizeof(void *) >= 8)
        setMaxMemoryUse(static_cast<int64_t>(4) * 1024 * 1024 * 1024);
}

///////////////

VSPlaneData::VSPlaneData(size_t dataSize, MemoryUse &mem) noexcept : refcount(1), mem(mem), size(dataSize + 2 * VSFrame::guardSpace) {
    data = vsh_aligned_malloc<uint8_t>(size + 2 * VSFrame::guardSpace, VSFrame::alignment);
    assert(data);
    if (!data)
        VS_FATAL_ERROR("Failed to allocate memory for plane. Out of memory.");

    mem.add(size);
#ifdef VS_FRAME_GUARD
    for (size_t i = 0; i < VSFrame::guardSpace / sizeof(VS_FRAME_GUARD_PATTERN); i++) {
        reinterpret_cast<uint32_t *>(data)[i] = VS_FRAME_GUARD_PATTERN;
        reinterpret_cast<uint32_t *>(data + size - VSFrame::guardSpace)[i] = VS_FRAME_GUARD_PATTERN;
    }
#endif
}

VSPlaneData::VSPlaneData(const VSPlaneData &d) noexcept : refcount(1), mem(d.mem), size(d.size) {
    data = vsh_aligned_malloc<uint8_t>(size, VSFrame::alignment);
    assert(data);
    if (!data)
        VS_FATAL_ERROR("Failed to allocate memory for plane in copy constructor. Out of memory.");

    mem.add(size);
    memcpy(data, d.data, size);
}

VSPlaneData::~VSPlaneData() {
    vsh_aligned_free(data);
    mem.subtract(size);
}

bool VSPlaneData::unique() noexcept {
    return (refcount == 1);
}

void VSPlaneData::add_ref() noexcept {
    ++refcount;
}

void VSPlaneData::release() noexcept {
    if (!--refcount)
        delete this;
}

///////////////

VSFrame::VSFrame(const VSVideoFormat &f, int width, int height, const VSFrame *propSrc, VSCore *core) noexcept : refcount(1), contentType(mtVideo), v3format(nullptr), width(width), height(height), properties(propSrc ? &propSrc->properties : nullptr), core(core) {
    if (width <= 0 || height <= 0)
        core->logFatal("Error in frame creation: dimensions are negative (" + std::to_string(width) + "x" + std::to_string(height) + ")");

    format.vf = f;
    numPlanes = format.vf.numPlanes;

    stride[0] = (width * (f.bytesPerSample) + (alignment - 1)) & ~(alignment - 1);

    if (numPlanes == 3) {
        int plane23 = ((width >> format.vf.subSamplingW) * (format.vf.bytesPerSample) + (alignment - 1)) & ~(alignment - 1);
        stride[1] = plane23;
        stride[2] = plane23;
    } else {
        stride[1] = 0;
        stride[2] = 0;
    }

    data[0] = new VSPlaneData(stride[0] * height, *core->memory);
    if (numPlanes == 3) {
        size_t size23 = stride[1] * (height >> format.vf.subSamplingH);
        data[1] = new VSPlaneData(size23, *core->memory);
        data[2] = new VSPlaneData(size23, *core->memory);
    }
}

VSFrame::VSFrame(const VSVideoFormat &f, int width, int height, const VSFrame * const *planeSrc, const int *plane, const VSFrame *propSrc, VSCore *core) noexcept : refcount(1), contentType(mtVideo), v3format(nullptr), width(width), height(height), properties(propSrc ? &propSrc->properties : nullptr), core(core) {
    if (width <= 0 || height <= 0)
        core->logFatal("Error in frame creation: dimensions are negative " + std::to_string(width) + "x" + std::to_string(height));

    format.vf = f;
    numPlanes = format.vf.numPlanes;

    stride[0] = (width * (format.vf.bytesPerSample) + (alignment - 1)) & ~(alignment - 1);

    if (numPlanes == 3) {
        int plane23 = ((width >> format.vf.subSamplingW) * (format.vf.bytesPerSample) + (alignment - 1)) & ~(alignment - 1);
        stride[1] = plane23;
        stride[2] = plane23;
    } else {
        stride[1] = 0;
        stride[2] = 0;
    }

    for (int i = 0; i < numPlanes; i++) {
        if (planeSrc[i]) {
            if (plane[i] < 0 || plane[i] >= planeSrc[i]->format.vf.numPlanes)
                core->logFatal("Error in frame creation: plane " + std::to_string(plane[i]) + " does not exist in the source frame");
            if (planeSrc[i]->getHeight(plane[i]) != getHeight(i) || planeSrc[i]->getWidth(plane[i]) != getWidth(i))
                core->logFatal("Error in frame creation: dimensions of plane " + std::to_string(plane[i]) + " do not match. Source: " + std::to_string(planeSrc[i]->getWidth(plane[i])) + "x" + std::to_string(planeSrc[i]->getHeight(plane[i])) + "; destination: " + std::to_string(getWidth(i)) + "x" + std::to_string(getHeight(i)));
            data[i] = planeSrc[i]->data[plane[i]];
            data[i]->add_ref();
        } else {
            if (i == 0) {
                data[i] = new VSPlaneData(stride[i] * height, *core->memory);
            } else {
                data[i] = new VSPlaneData(stride[i] * (height >> format.vf.subSamplingH), *core->memory);
            }
        }
    }
}

VSFrame::VSFrame(const VSAudioFormat &f, int numSamples, const VSFrame *propSrc, VSCore *core) noexcept
    : refcount(1), contentType(mtAudio), v3format(nullptr), properties(propSrc ? &propSrc->properties : nullptr), core(core) {
    if (numSamples <= 0)
        core->logFatal("Error in frame creation: bad number of samples (" + std::to_string(numSamples) + ")");
    
    format.af = f;
    numPlanes = format.af.numChannels;

    width = numSamples;

    stride[0] = format.af.bytesPerSample * VS_AUDIO_FRAME_SAMPLES;

    data[0] = new VSPlaneData(stride[0] * format.af.numChannels, *core->memory);
}

VSFrame::VSFrame(const VSAudioFormat &f, int numSamples, const VSFrame * const *channelSrc, const int *channel, const VSFrame *propSrc, VSCore *core) noexcept
    : refcount(1), contentType(mtAudio), v3format(nullptr), properties(propSrc ? &propSrc->properties : nullptr), core(core) {
    if (numSamples <= 0)
        core->logFatal("Error in frame creation: bad number of samples (" + std::to_string(numSamples) + ")");

    format.af = f;
    numPlanes = format.af.numChannels;

    width = numSamples;

    stride[0] = format.af.bytesPerSample * VS_AUDIO_FRAME_SAMPLES;

    data[0] = new VSPlaneData(stride[0] * format.af.numChannels, *core->memory);

    for (int i = 0; i < numPlanes; i++) {
        if (channelSrc[i]) {
            if (channel[i] < 0 || channel[i] >= channelSrc[i]->format.af.numChannels)
                core->logFatal("Error in frame creation: channel " + std::to_string(channel[i]) + " does not exist in the source frame");
            if (channelSrc[i]->getFrameLength() != getFrameLength())
                core->logFatal("Error in frame creation: length of frame does not match. Source: " + std::to_string(channelSrc[i]->getFrameLength()) + "; destination: " + std::to_string(getFrameLength()));
            memcpy(getWritePtr(i), channelSrc[i]->getReadPtr(channel[i]), getFrameLength() * format.af.bytesPerSample);
        }
    }
}

VSFrame::VSFrame(const VSFrame &f) noexcept : refcount(1), v3format(nullptr) {
    contentType = f.contentType;
    data[0] = f.data[0];
    data[1] = f.data[1];
    data[2] = f.data[2];
    data[0]->add_ref();
    if (data[1]) {
        data[1]->add_ref();
        data[2]->add_ref();
    }
    format = f.format;
    numPlanes = f.numPlanes;
    width = f.width;
    height = f.height;
    stride[0] = f.stride[0];
    stride[1] = f.stride[1];
    stride[2] = f.stride[2];
    properties = f.properties;
    core = f.core;
}

VSFrame::~VSFrame() {
    data[0]->release();
    if (data[1]) {
        data[1]->release();
        data[2]->release();
    }
}

const vs3::VSVideoFormat *VSFrame::getVideoFormatV3() const noexcept {
    assert(contentType == mtVideo);
    if (!v3format)
        v3format = core->VideoFormatToV3(format.vf);
    return v3format;
}

ptrdiff_t VSFrame::getStride(int plane) const {
    assert(contentType == mtVideo);
    if (plane < 0 || plane >= numPlanes)
        return 0;
    return stride[plane];
}

const uint8_t *VSFrame::getReadPtr(int plane) const {
    if (plane < 0 || plane >= numPlanes)
        return nullptr;

    if (contentType == mtVideo)
        return data[plane]->data + guardSpace;
    else
        return data[0]->data + guardSpace + plane * stride[0];
}

uint8_t *VSFrame::getWritePtr(int plane) {
    if (plane < 0 || plane >= numPlanes)
        return nullptr;

    // copy the plane data if this isn't the only reference
    if (contentType == mtVideo) {
        if (!data[plane]->unique()) {
            VSPlaneData *old = data[plane];
            data[plane] = new VSPlaneData(*data[plane]);
            old->release();
        }

        return data[plane]->data + guardSpace;
    } else {
        if (!data[0]->unique()) {
            VSPlaneData *old = data[0];
            data[0] = new VSPlaneData(*data[0]);
            old->release();
        }

        return data[0]->data + guardSpace + plane * stride[0];
    }
}

#ifdef VS_FRAME_GUARD
bool VSFrame::verifyGuardPattern() const {
    for (int p = 0; p < ((contentType == mtVideo) ? numPlanes : 1); p++) {
        for (size_t i = 0; i < guardSpace / sizeof(VS_FRAME_GUARD_PATTERN); i++) {
            uint32_t p1 = reinterpret_cast<uint32_t *>(data[p]->data)[i];
            uint32_t p2 = reinterpret_cast<uint32_t *>(data[p]->data + data[p]->size - guardSpace)[i];
            if (reinterpret_cast<uint32_t *>(data[p]->data)[i] != VS_FRAME_GUARD_PATTERN ||
                reinterpret_cast<uint32_t *>(data[p]->data + data[p]->size - guardSpace)[i] != VS_FRAME_GUARD_PATTERN)
                return false;
        }
    }

    return true;
}
#endif

struct split1 {
    enum empties_t { empties_ok, no_empties };
};

template <typename Container>
Container& split(
    Container& result,
    const typename Container::value_type& s,
    const typename Container::value_type& delimiters,
    split1::empties_t empties = split1::empties_ok) {
    result.clear();
    size_t current;
    size_t next = -1;
    do {
        if (empties == split1::no_empties) {
            next = s.find_first_not_of(delimiters, next + 1);
            if (next == Container::value_type::npos) break;
            next -= 1;
        }
        current = next + 1;
        next = s.find_first_of(delimiters, current);
        result.push_back(s.substr(current, next - current));
    } while (next != Container::value_type::npos);
    return result;
}

void VSPluginFunction::parseArgString(const std::string &argString, std::vector<FilterArgument> &argsOut, int apiMajor) {
    std::vector<std::string> argList;
    split(argList, argString, std::string(";"), split1::no_empties);

    argsOut.reserve(argList.size());
    for (const std::string &arg : argList) {
        std::vector<std::string> argParts;
        split(argParts, arg, std::string(":"), split1::no_empties);

        if (argParts.size() == 1 && argParts[0] == "any") {
            argsOut.emplace_back("", ptUnset, false, false, false);
            continue;
        }

        if (argParts.size() < 2)
            throw std::runtime_error("Invalid argument specifier '" + arg + "'. It appears to be incomplete.");

        bool arr = false;
        bool opt = false;
        bool empty = false;

        VSPropertyType type = ptUnset;
        const std::string &argName = argParts[0];
        std::string &typeName = argParts[1];

        if (typeName.length() > 2 && typeName.substr(typeName.length() - 2) == "[]") {
            typeName.resize(typeName.length() - 2);
            arr = true;
        }

        if (typeName == "int") {
            type = ptInt;
        } else if (typeName == "float") {
            type = ptFloat;
        } else if (typeName == "data") {
            type = ptData;
        } else if ((typeName == "vnode" && apiMajor > VAPOURSYNTH3_API_MAJOR) || (apiMajor == VAPOURSYNTH3_API_MAJOR && typeName == "clip")) {
            type = ptVideoNode;
        } else if (typeName == "anode" && apiMajor > VAPOURSYNTH3_API_MAJOR) {
            type = ptAudioNode;
        } else if ((typeName == "vframe" && apiMajor > VAPOURSYNTH3_API_MAJOR) || (apiMajor == VAPOURSYNTH3_API_MAJOR && typeName == "frame")) {
            type = ptVideoFrame;
        } else if (typeName == "aframe" && apiMajor > VAPOURSYNTH3_API_MAJOR) {
            type = ptAudioFrame;
        } else if (typeName == "func") {
            type = ptFunction;
        } else {
            throw std::runtime_error("Argument '" + argName + "' has invalid type '" + typeName + "'.");
        }

        for (size_t i = 2; i < argParts.size(); i++) {
            if (argParts[i] == "opt") {
                if (opt)
                    throw std::runtime_error("Argument '" + argName + "' has duplicate argument specifier '" + argParts[i] + "'");
                opt = true;
            } else if (argParts[i] == "empty") {
                if (empty)
                    throw std::runtime_error("Argument '" + argName + "' has duplicate argument specifier '" + argParts[i] + "'");
                empty = true;
            } else {
                throw std::runtime_error("Argument '" + argName + "' has unknown argument modifier '" + argParts[i] + "'");
            }
        }

        if (!isValidIdentifier(argName))
            throw std::runtime_error("Argument name '" + argName + "' contains illegal characters.");

        if (empty && !arr)
            throw std::runtime_error("Argument '" + argName + "' is not an array. Only array arguments can have the empty flag set.");

        argsOut.emplace_back(argName, type, arr, empty, opt);
    }
}

VSPluginFunction::VSPluginFunction(const std::string &name, const std::string &argString, const std::string &returnType, VSPublicFunction func, void *functionData, VSPlugin *plugin)
    : func(func), functionData(functionData), plugin(plugin), name(name), argString(argString), returnType(returnType) {
    parseArgString(argString, inArgs, plugin->apiMajor);
    if (plugin->apiMajor == 3)
        this->argString = getV4ArgString(); // construct to V4 equivalent arg string
    if (returnType != "any")
        parseArgString(returnType, retArgs, plugin->apiMajor);
}

VSMap *VSPluginFunction::invoke(const VSMap &args) {
    VSMap *v = new VSMap;

    try {
        std::set<std::string> remainingArgs;
        for (size_t i = 0; i < args.size(); i++)
            remainingArgs.insert(args.key(i));

        for (const FilterArgument &fa : inArgs) {
            // ptUnset as an argument type means any value is accepted beyond the declared ones
            if (fa.type == ptUnset) {
                // this will always be the last argument, therefore it's safe to clear out the remaining ones since they'll
                // already have been checked
                remainingArgs.clear();
                continue;
            }

            int propType = vs_internal_vsapi.mapGetType(&args, fa.name.c_str());

            if (propType != ptUnset) {
                remainingArgs.erase(fa.name);

                if (fa.type != propType)
                    throw VSException(name + ": argument " + fa.name + " is not of the correct type");

                VSArrayBase *arr = args.find(fa.name);

                if (!fa.arr && arr->size() > 1)
                    throw VSException(name + ": argument " + fa.name + " is not of array type but more than one value was supplied");

                if (!fa.empty && arr->size() < 1)
                    throw VSException(name + ": argument " + fa.name + " does not accept empty arrays");

            } else if (!fa.opt) {
                throw VSException(name + ": argument " + fa.name + " is required");
            }
        }

        if (!remainingArgs.empty()) {
            auto iter = remainingArgs.cbegin();
            std::string s = *iter;
            ++iter;
            for (; iter != remainingArgs.cend(); ++iter)
                s += ", " + *iter;
            throw VSException(name + ": no argument(s) named " + s);
        }

        bool enableGraphInspection = plugin->core->enableGraphInspection;
        if (enableGraphInspection) {
            plugin->core->functionFrame = std::make_shared<VSFunctionFrame>(name, new VSMap(&args), plugin->core->functionFrame);
        }
        func(&args, v, functionData, plugin->core, getVSAPIInternal(plugin->apiMajor));
        if (enableGraphInspection) {
            assert(plugin->core->functionFrame);
            plugin->core->functionFrame = plugin->core->functionFrame->next;
        }

        if (plugin->apiMajor == VAPOURSYNTH3_API_MAJOR && !args.isV3Compatible())
            plugin->core->logFatal(name + ": filter node returned not yet supported type");

    } catch (VSException &e) {
        vs_internal_vsapi.mapSetError(v, e.what());
    }

    return v;
}

bool VSPluginFunction::isV3Compatible() const {
    for (const auto &iter : inArgs)
        if (iter.type == ptAudioNode || iter.type == ptAudioFrame)
            return false;
    for (const auto &iter : retArgs)
        if (iter.type == ptAudioNode || iter.type == ptAudioFrame)
            return false;
    return true;
}

std::string VSPluginFunction::getV4ArgString() const {
    std::string tmp;
    for (const auto &iter : inArgs) {
        assert(iter.type != ptAudioNode && iter.type != ptAudioFrame);

        tmp += iter.name + ":";

        switch (iter.type) {
            case ptInt:
                tmp += "int"; break;
            case ptFloat:
                tmp += "float"; break;
            case ptData:
                tmp += "data"; break;
            case ptVideoNode:
                tmp += "vnode"; break;
            case ptVideoFrame:
                tmp += "vframe"; break;
            case ptFunction:
                tmp += "func"; break;
            default:
                assert(false);
        }
        if (iter.arr)
            tmp += "[]";
        if (iter.opt)
            tmp += ":opt";
        if (iter.empty)
            tmp += ":empty";
        tmp += ";";
    }
    return tmp;
}

std::string VSPluginFunction::getV3ArgString() const {
    std::string tmp;
    for (const auto &iter : inArgs) {
        assert(iter.type != ptAudioNode && iter.type != ptAudioFrame);

        tmp += iter.name + ":";

        switch (iter.type) {
            case ptInt:
                tmp += "int"; break;
            case ptFloat:
                tmp += "float"; break;
            case ptData:
                tmp += "data"; break;
            case ptVideoNode:
                tmp += "clip"; break;
            case ptVideoFrame:
                tmp += "frame"; break;
            case ptFunction:
                tmp += "func"; break;
            default:
                assert(false);
        }
        if (iter.arr)
            tmp += "[]";
        if (iter.opt)
            tmp += ":opt";
        if (iter.empty)
            tmp += ":empty";
        tmp += ";";
    }
    return tmp;
}

const std::string &VSPluginFunction::getName() const {
    return name;
}

const std::string &VSPluginFunction::getArguments() const {
    return argString;
}

const std::string &VSPluginFunction::getReturnType() const {
    return returnType;
}

struct MakeLinearWrapper {
    vs3::VSFilterGetFrame getFrameFunc;
    VSFilterFree freeFunc;
    void *instanceData;
    int threshold;
    int lastFrame = -1;

    MakeLinearWrapper(VSFilterGetFrame g, VSFilterFree f, void *instanceData, int threshold) :
        getFrameFunc(reinterpret_cast<vs3::VSFilterGetFrame>(g)),
        freeFunc(f), instanceData(instanceData), threshold(threshold) {}

    static const VSFrame *VS_CC getFrame(int n, int activationReason, void *instanceData, void **frameData, VSFrameContext *frameCtx, VSCore *core, const VSAPI *vsapi);
    static void VS_CC freeFilter(void *instanceData, VSCore *core, const VSAPI *vsapi);
};

VSNode::VSNode(const VSMap *in, VSMap *out, const std::string &name, vs3::VSFilterInit init, VSFilterGetFrame getFrame, VSFilterFree freeFunc, VSFilterMode filterMode, int flags, void *instanceData, int apiMajor, VSCore *core) :
    refcount(1), nodeType(mtVideo), instanceData(instanceData), name(name), filterGetFrame(getFrame), freeFunc(freeFunc), filterMode(filterMode), apiMajor(apiMajor), core(core), serialFrame(-1), processingTime(0) {

    if (flags & ~(vs3::nfNoCache | vs3::nfIsCache | vs3::nfMakeLinear))
        throw VSException("Filter " + name  + " specified unknown flags");

    if ((flags & vs3::nfIsCache) && !(flags & vs3::nfNoCache))
        throw VSException("Filter " + name + " specified an illegal combination of flags (nfNoCache must always be set with nfIsCache)");


    VSMap inval(in);
    init(&inval, out, &this->instanceData, this, core, reinterpret_cast<const vs3::VSAPI3 *>(getVSAPIInternal(3)));

    if (out->hasError()) {
        throw VSException(vs_internal_vsapi.mapGetError(out));
    }

    if (vi.format.colorFamily == 0) {
        throw VSException("Filter " + name + " didn't set videoinfo");
    }

    if (vi.numFrames <= 0) {
        throw VSException("Filter " + name + " returned zero or negative frame count");
    }

    core->filterInstanceCreated();

    // Scan the in map for clips, these are probably the real dependencies
    // Worst case there are false positives and an extra cache gets activated
    // NoCache is generally the equivalent strict spatial for filters

    int requestPattern = !!(flags & vs3::nfNoCache) ? rpNoFrameReuse : rpGeneral;
    int numKeys = vs_internal_vsapi.mapNumKeys(in);

    bool makeLinear = (flags & vs3::nfMakeLinear);
    bool hasVideoNodes = false;
    for (int i = 0; i < numKeys; i++) {
        const char *key = vs_internal_vsapi.mapGetKey(in, i);
        if (vs_internal_vsapi.mapGetType(in, key) == ptVideoNode) {
            int numElems = vs_internal_vsapi.mapNumElements(in, key);
            for (int j = 0; j < numElems; j++) {
                hasVideoNodes = true;
                VSNode *sn = vs_internal_vsapi.mapGetNode(in, key, j, nullptr);
                this->dependencies.push_back({sn, requestPattern});
                sn->addConsumer(this, requestPattern);
            }
        }
    }

    if (makeLinear && !hasVideoNodes) {
        this->apiMajor = 4;
        MakeLinearWrapper *wrapper = new MakeLinearWrapper(filterGetFrame, freeFunc, instanceData, setLinear());
        filterGetFrame = MakeLinearWrapper::getFrame;
        this->freeFunc = MakeLinearWrapper::freeFilter;
        this->instanceData = reinterpret_cast<void *>(wrapper);
    }

    if (core->enableGraphInspection) {
        functionFrame = core->functionFrame;
    }
}

VSNode::VSNode(const std::string &name, const VSVideoInfo *vi, VSFilterGetFrame getFrame, VSFilterFree freeFunc, VSFilterMode filterMode, const VSFilterDependency *dependencies, int numDeps, void *instanceData, int apiMajor, VSCore *core) :
    refcount(1), nodeType(mtVideo), instanceData(instanceData), name(name), filterGetFrame(getFrame), freeFunc(freeFunc), filterMode(filterMode), apiMajor(apiMajor), core(core), serialFrame(-1), processingTime(0) {

    if (!core->isValidVideoInfo(*vi))
        throw VSException("The VSVideoInfo structure passed by " + name + " is invalid.");

    this->vi = *vi;
    this->v3vi = core->VideoInfoToV3(*vi);

    core->filterInstanceCreated();

    this->dependencies.reserve(numDeps);
    for (int i = 0; i < numDeps; i++) {
        this->dependencies.push_back(dependencies[i]);
        dependencies[i].source->add_ref();
        dependencies[i].source->addConsumer(this, dependencies[i].requestPattern);
    }

    if (core->enableGraphInspection) {
        functionFrame = core->functionFrame;
    }
}

VSNode::VSNode(const std::string &name, const VSAudioInfo *ai, VSFilterGetFrame getFrame, VSFilterFree freeFunc, VSFilterMode filterMode, const VSFilterDependency *dependencies, int numDeps, void *instanceData, int apiMajor, VSCore *core) :
    refcount(1), nodeType(mtAudio), instanceData(instanceData), name(name), filterGetFrame(getFrame), freeFunc(freeFunc), filterMode(filterMode), apiMajor(apiMajor), core(core), serialFrame(-1), processingTime(0) {

    if (!core->isValidAudioInfo(*ai))
{'fraction_non_alphanumeric': 0.14124585203143, 'fraction_numerical': 0.010127061089045925, 'mean_word_length': 2.892778261160565, 'pattern_counts': {'":': 26, '<': 125, '<?xml version=': 0, '>': 327, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3995098', 'n_tokens_mistral': 18971, 'n_tokens_neox': 18119, 'n_words': 4437}