<commit_before>#include "Base.h"
#include "Bundle.h"
#include "FileSystem.h"
#include "MeshPart.h"
#include "Scene.h"
#include "Joint.h"

#define BUNDLE_VERSION_MAJOR            1
#define BUNDLE_VERSION_MINOR            2

#define BUNDLE_TYPE_SCENE               1
#define BUNDLE_TYPE_NODE                2
#define BUNDLE_TYPE_ANIMATIONS          3
#define BUNDLE_TYPE_ANIMATION           4
#define BUNDLE_TYPE_ANIMATION_CHANNEL   5
#define BUNDLE_TYPE_MODEL               10
#define BUNDLE_TYPE_MATERIAL            16
#define BUNDLE_TYPE_EFFECT              18
#define BUNDLE_TYPE_CAMERA              32
#define BUNDLE_TYPE_LIGHT               33
#define BUNDLE_TYPE_MESH                34
#define BUNDLE_TYPE_MESHPART            35
#define BUNDLE_TYPE_MESHSKIN            36
#define BUNDLE_TYPE_FONT                128

// For sanity checking string reads
#define BUNDLE_MAX_STRING_LENGTH        5000

namespace gameplay
{

static std::vector<Bundle*> __bundleCache;

Bundle::Bundle(const char* path) :
    _path(path), _referenceCount(0), _references(NULL), _file(NULL), _trackedNodes(NULL)
{
}

Bundle::~Bundle()
{
    clearLoadSession();

    // Remove this Bundle from the cache.
    std::vector<Bundle*>::iterator itr = std::find(__bundleCache.begin(), __bundleCache.end(), this);
    if (itr != __bundleCache.end())
    {
        __bundleCache.erase(itr);
    }

    SAFE_DELETE_ARRAY(_references);

    if (_file)
    {
        fclose(_file);
        _file = NULL;
    }
}

template <class T>
bool Bundle::readArray(unsigned int* length, T** ptr)
{
    GP_ASSERT(length);
    GP_ASSERT(ptr);
    GP_ASSERT(_file);

    if (!read(length))
    {
        GP_ERROR("Failed to read the length of an array of data (to be read into an array).");
        return false;
    }
    if (*length > 0)
    {
        *ptr = new T[*length];
        if (fread(*ptr, sizeof(T), *length, _file) != *length)
        {
            GP_ERROR("Failed to read an array of data from bundle (into an array).");
            SAFE_DELETE_ARRAY(*ptr);
            return false;
        }
    }
    return true;
}

template <class T>
bool Bundle::readArray(unsigned int* length, std::vector<T>* values)
{
    GP_ASSERT(length);
    GP_ASSERT(_file);

    if (!read(length))
    {
        GP_ERROR("Failed to read the length of an array of data (to be read into a std::vector).");
        return false;
    }
    if (*length > 0 && values)
    {
        values->resize(*length);
        if (fread(&(*values)[0], sizeof(T), *length, _file) != *length)
        {
            GP_ERROR("Failed to read an array of data from bundle (into a std::vector).");
            return false;
        }
    }
    return true;
}

template <class T>
bool Bundle::readArray(unsigned int* length, std::vector<T>* values, unsigned int readSize)
{
    GP_ASSERT(length);
    GP_ASSERT(_file);
    GP_ASSERT(sizeof(T) >= readSize);

    if (!read(length))
    {
        GP_ERROR("Failed to read the length of an array of data (to be read into a std::vector with a specified single element read size).");
        return false;
    }
    if (*length > 0 && values)
    {
        values->resize(*length);
        if (fread(&(*values)[0], readSize, *length, _file) != *length)
        {
            GP_ERROR("Failed to read an array of data from bundle (into a std::vector with a specified single element read size).");
            return false;
        }
    }
    return true;
}

std::string readString(FILE* fp)
{
    GP_ASSERT(fp);

    unsigned int length;
    if (fread(&length, 4, 1, fp) != 1)
    {
        GP_ERROR("Failed to read the length of a string from a bundle.");
        return std::string();
    }

    // Sanity check to detect if string length is far too big.
    GP_ASSERT(length < BUNDLE_MAX_STRING_LENGTH);

    std::string str;
    if (length > 0)
    {
        str.resize(length);
        if (fread(&str[0], 1, length, fp) != length)
        {
            GP_ERROR("Failed to read string from bundle.");
            return std::string();
        }
    }
    return str;
}

Bundle* Bundle::create(const char* path)
{
    GP_ASSERT(path);

    // Search the cache for this bundle.
    for (unsigned int i = 0, count = __bundleCache.size(); i < count; ++i)
    {
        Bundle* p = __bundleCache[i];
        GP_ASSERT(p);
        if (p->_path == path)
        {
            // Found a match
            p->addRef();
            return p;
        }
    }

    // Open the bundle.
    FILE* fp = FileSystem::openFile(path, "rb");
    if (!fp)
    {
        GP_ERROR("Failed to open file '%s'.", path);
        return NULL;
    }

    // Read the GPB header info.
    char sig[9];
    if (fread(sig, 1, 9, fp) != 9 || memcmp(sig, "GPB\r\n\x1A\n", 9) != 0)
    {
        GP_ERROR("Invalid GPB header for bundle '%s'.", path);
        if (fclose(fp) != 0)<commit_msg>Fix for international(non-Latin) build environment<commit_after>#include "Base.h"
#include "Bundle.h"
#include "FileSystem.h"
#include "MeshPart.h"
#include "Scene.h"
#include "Joint.h"

#define BUNDLE_VERSION_MAJOR            1
#define BUNDLE_VERSION_MINOR            2

#define BUNDLE_TYPE_SCENE               1
#define BUNDLE_TYPE_NODE                2
#define BUNDLE_TYPE_ANIMATIONS          3
#define BUNDLE_TYPE_ANIMATION           4
#define BUNDLE_TYPE_ANIMATION_CHANNEL   5
#define BUNDLE_TYPE_MODEL               10
#define BUNDLE_TYPE_MATERIAL            16
#define BUNDLE_TYPE_EFFECT              18
#define BUNDLE_TYPE_CAMERA              32
#define BUNDLE_TYPE_LIGHT               33
#define BUNDLE_TYPE_MESH                34
#define BUNDLE_TYPE_MESHPART            35
#define BUNDLE_TYPE_MESHSKIN            36
#define BUNDLE_TYPE_FONT                128

// For sanity checking string reads
#define BUNDLE_MAX_STRING_LENGTH        5000

namespace gameplay
{

static std::vector<Bundle*> __bundleCache;

Bundle::Bundle(const char* path) :
    _path(path), _referenceCount(0), _references(NULL), _file(NULL), _trackedNodes(NULL)
{
}

Bundle::~Bundle()
{
    clearLoadSession();

    // Remove this Bundle from the cache.
    std::vector<Bundle*>::iterator itr = std::find(__bundleCache.begin(), __bundleCache.end(), this);
    if (itr != __bundleCache.end())
    {
        __bundleCache.erase(itr);
    }

    SAFE_DELETE_ARRAY(_references);

    if (_file)
    {
        fclose(_file);
        _file = NULL;
    }
}

template <class T>
bool Bundle::readArray(unsigned int* length, T** ptr)
{
    GP_ASSERT(length);
    GP_ASSERT(ptr);
    GP_ASSERT(_file);

    if (!read(length))
    {
        GP_ERROR("Failed to read the length of an array of data (to be read into an array).");
        return false;
    }
    if (*length > 0)
    {
        *ptr = new T[*length];
        if (fread(*ptr, sizeof(T), *length, _file) != *length)
        {
            GP_ERROR("Failed to read an array of data from bundle (into an array).");
            SAFE_DELETE_ARRAY(*ptr);
            return false;
        }
    }
    return true;
}

template <class T>
bool Bundle::readArray(unsigned int* length, std::vector<T>* values)
{
    GP_ASSERT(length);
    GP_ASSERT(_file);

    if (!read(length))
    {
        GP_ERROR("Failed to read the length of an array of data (to be read into a std::vector).");
        return false;
    }
    if (*length > 0 && values)
    {
        values->resize(*length);
        if (fread(&(*values)[0], sizeof(T), *length, _file) != *length)
        {
            GP_ERROR("Failed to read an array of data from bundle (into a std::vector).");
            return false;
        }
    }
    return true;
}

template <class T>
bool Bundle::readArray(unsigned int* length, std::vector<T>* values, unsigned int readSize)
{
    GP_ASSERT(length);
    GP_ASSERT(_file);
    GP_ASSERT(sizeof(T) >= readSize);

    if (!read(length))
    {
        GP_ERROR("Failed to read the length of an array of data (to be read into a std::vector with a specified single element read size).");
        return false;
    }
    if (*length > 0 && values)
    {
        values->resize(*length);
        if (fread(&(*values)[0], readSize, *length, _file) != *length)
        {
            GP_ERROR("Failed to read an array of data from bundle (into a std::vector with a specified single element read size).");
            return false;
        }
    }
    return true;
}

std::string readString(FILE* fp)
{
    GP_ASSERT(fp);

    unsigned int length;
    if (fread(&length, 4, 1, fp) != 1)
    {
        GP_ERROR("Failed to read the length of a string from a bundle.");
        return std::string();
    }

    // Sanity check to detect if string length is far too big.
    GP_ASSERT(length < BUNDLE_MAX_STRING_LENGTH);

    std::string str;
    if (length > 0)
    {
        str.resize(length);
        if (fread(&str[0], 1, length, fp) != length)
        {
            GP_ERROR("Failed to read string from bundle.");
            return std::string();
        }
    }
    return str;
}

Bundle* Bundle::create(const char* path)
{
    GP_ASSERT(path);

    // Search the cache for this bundle.
    for (unsigned int i = 0, count = __bundleCache.size(); i < count; ++i)
    {
        Bundle* p = __bundleCache[i];
        GP_ASSERT(p);
        if (p->_path == path)
        {
            // Found a match
            p->addRef();
            return p;
        }
    }

    // Open the bundle.
    FILE* fp = FileSystem::openFile(path, "rb");
    if (!fp)
    {
        GP_ERROR("Failed to open file '%s'.", path);
        return NULL;
    }

    // Read the GPB header info.
    char sig[9];
    if (fread(sig, 1, 9, fp) != 9 || memcmp(sig, "\xABGPB\xBB\r\n\x1A\n", 9) != 0)
    {
        GP_ERROR("Invalid GPB header for bundle '%s'.", path);
        if (fclose(fp) != 0)
{'fraction_non_alphanumeric': 0.12339199341360502, 'fraction_numerical': 0.010497066995986416, 'mean_word_length': 2.2588866532528504, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5864957', 'n_tokens_mistral': 3196, 'n_tokens_neox': 3129, 'n_words': 881}