<commit_before>/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * BP4Reader.cpp
 *
 *  Created on: Aug 1, 2018
 *      Author: <NAME> <EMAIL>
 */

#include "BP4Reader.h"
#include "BP4Reader.tcc"

#include "adios2/toolkit/profiling/taustubs/tautimer.hpp"

#include <chrono>
#include <errno.h>

namespace adios2
{
namespace core
{
namespace engine
{

BP4Reader::BP4Reader(IO &io, const std::string &name, const Mode mode,
                     helper::Comm comm)
: Engine("BP4Reader", io, name, mode, std::move(comm)),
  m_BP4Deserializer(m_Comm), m_MDFileManager(m_Comm), m_DataFileManager(m_Comm),
  m_MDIndexFileManager(m_Comm), m_ActiveFlagFileManager(m_Comm)
{
    TAU_SCOPED_TIMER("BP4Reader::Open");
    Init();
}

StepStatus BP4Reader::BeginStep(StepMode mode, const float timeoutSeconds)
{
    TAU_SCOPED_TIMER("BP4Reader::BeginStep");
    if (mode != StepMode::Read)
    {
        throw std::invalid_argument("ERROR: mode is not supported yet, "
                                    "only Read is valid for "
                                    "engine BP4Reader, in call to "
                                    "BeginStep\n");
    }

    if (!m_BP4Deserializer.m_DeferredVariables.empty())
    {
        throw std::invalid_argument(
            "ERROR: existing variables subscribed with "
            "GetDeferred, did you forget to call "
            "PerformGets() or EndStep()?, in call to BeginStep\n");
    }

    // used to inquire for variables in streaming mode
    m_IO.m_ReadStreaming = true;
    StepStatus status = StepStatus::OK;

    if (m_FirstStep)
    {
        if (m_BP4Deserializer.m_MetadataSet.StepsCount == 0)
        {
            status = CheckForNewSteps(Seconds(timeoutSeconds));
        }
    }
    else
    {
        if (m_CurrentStep + 1 >= m_BP4Deserializer.m_MetadataSet.StepsCount)
        {
            status = CheckForNewSteps(Seconds(timeoutSeconds));
        }
    }

    // This should be after getting new steps

    if (status == StepStatus::OK)
    {
        if (m_FirstStep)
        {
            m_FirstStep = false;
        }
        else
        {
            ++m_CurrentStep;
        }

        m_IO.m_EngineStep = m_CurrentStep;
        m_IO.ResetVariablesStepSelection(false,
                                         "in call to BP4 Reader BeginStep");

        // caches attributes for each step
        // if a variable name is a prefix
        // e.g. var  prefix = {var/v1, var/v2, var/v3}
        m_IO.SetPrefixedNames(true);
    }

    return status;
}

size_t BP4Reader::CurrentStep() const { return m_CurrentStep; }

void BP4Reader::EndStep()
{
    TAU_SCOPED_TIMER("BP4Reader::EndStep");
    PerformGets();
}

void BP4Reader::PerformGets()
{
    TAU_SCOPED_TIMER("BP4Reader::PerformGets");
    if (m_BP4Deserializer.m_DeferredVariables.empty())
    {
        return;
    }

    for (const std::string &name : m_BP4Deserializer.m_DeferredVariables)
    {
        const DataType type = m_IO.InquireVariableType(name);

        if (type == DataType::Compound)
        {
        }
#define declare_type(T)                                                        \
    else if (type == helper::GetDataType<T>())                                 \
    {                                                                          \
        Variable<T> &variable =                                                \
            FindVariable<T>(name, "in call to PerformGets, EndStep or Close"); \
        for (auto &blockInfo : variable.m_BlocksInfo)                          \
        {                                                                      \
            m_BP4Deserializer.SetVariableBlockInfo(variable, blockInfo);       \
        }                                                                      \
        ReadVariableBlocks(variable);                                          \
        variable.m_BlocksInfo.clear();                                         \
    }
        ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
#undef declare_type
    }

    m_BP4Deserializer.m_DeferredVariables.clear();
}

// PRIVATE
void BP4Reader::Init()
{
    if (m_OpenMode != Mode::Read)
    {
        throw std::invalid_argument("ERROR: BPFileReader only "
                                    "supports OpenMode::Read from" +
                                    m_Name + " " + m_EndMessage);
    }

    m_BP4Deserializer.Init(m_IO.m_Parameters, "in call to BP4::Open to write");
    InitTransports();

    /* Do a collective wait for the file(s) to appear within timeout.
       Make sure every process comes to the same conclusion */
    const Seconds timeoutSeconds =
        Seconds(m_BP4Deserializer.m_Parameters.OpenTimeoutSecs);

    // set poll to 1/100 of timeout
    Seconds pollSeconds = timeoutSeconds / 100.0;
    static const auto pollSecondsMin = Seconds(1.0);
    if (pollSeconds < pollSecondsMin)
    {
        pollSeconds = pollSecondsMin;
    }
    static const auto pollSecondsMax = Seconds(10.0);
    if (pollSeconds > pollSecondsMax)
    {
        pollSeconds = pollSecondsMax;
    }

    TimePoint timeoutInstant =
        std::chrono::steady_clock::now() + timeoutSeconds;

    OpenFiles(timeoutInstant, pollSeconds, timeoutSeconds);
    if (!m_BP4Deserializer.m_Parameters.StreamReader)
    {
        /* non-stream reader gets as much steps as available now */
        InitBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds);
    }
}

bool BP4Reader::SleepOrQuit(const TimePoint &timeoutInstant,
                            const Seconds &pollSeconds)
{
    auto now = std::chrono::steady_clock::now();
    if (now + pollSeconds >= timeoutInstant)
    {
        return false;
    }
    auto remainderTime = timeoutInstant - now;
    auto sleepTime = pollSeconds;
    if (remainderTime < sleepTime)
    {
        sleepTime = remainderTime;
    }
    std::this_thread::sleep_for(sleepTime);
    return true;
}

size_t BP4Reader::OpenWithTimeout(transportman::TransportMan &tm,
                                  const std::vector<std::string> &fileNames,
                                  const TimePoint &timeoutInstant,
                                  const Seconds &pollSeconds,
                                  std::string &lasterrmsg /*INOUT*/)
{
    size_t flag = 1; // 0 = OK, opened file, 1 = timeout, 2 = error
    do
    {
        try
        {
            errno = 0;
            const bool profile = m_BP4Deserializer.m_Profiler.m_IsActive;
            tm.OpenFiles(fileNames, adios2::Mode::Read,
                         m_IO.m_TransportsParameters, profile);
            flag = 0; // found file
            break;
        }
        catch (std::ios_base::failure &e)
        {
            lasterrmsg =
                std::string("errno=" + std::to_string(errno) + ": " + e.what());
            if (errno == ENOENT)
            {
                flag = 1; // timeout
            }
            else
            {
                flag = 2; // fatal error
                break;
            }
        }
    } while (SleepOrQuit(timeoutInstant, pollSeconds));
    return flag;
}

void BP4Reader::OpenFiles(TimePoint &timeoutInstant, const Seconds &pollSeconds,
                          const Seconds &timeoutSeconds)
{
    /* Poll */
    size_t flag = 1; // 0 = OK, opened file, 1 = timeout, 2 = error
    std::string lasterrmsg;
    if (m_BP4Deserializer.m_RankMPI == 0)
    {
        /* Open the metadata index table */
        const std::string metadataIndexFile(
            m_BP4Deserializer.GetBPMetadataIndexFileName(m_Name));

        flag = OpenWithTimeout(m_MDIndexFileManager, {metadataIndexFile},
                               timeoutInstant, pollSeconds, lasterrmsg);
        if (flag == 0)
        {
            /* Open the metadata file */
            const std::string metadataFile(
                m_BP4Deserializer.GetBPMetadataFileName(m_Name));

            /* We found md.idx. If we don't find md.0 immediately  we should
             * wait a little bit hoping for the file system to catch up.
             * This slows down finding the error in file reading mode but
             * it will be more robust in streaming mode
             */
            if (timeoutSeconds == Seconds(0.0))
            {
                timeoutInstant += Seconds(5.0);
            }

            flag = OpenWithTimeout(m_MDFileManager, {metadataFile},
                                   timeoutInstant, pollSeconds, lasterrmsg);
            if (flag != 0)
            {
                /* Close the metadata index table */
                m_MDIndexFileManager.CloseFiles();
            }
        }
    }

    flag = m_Comm.BroadcastValue(flag, 0);
    if (flag == 2)
    {
        if (m_BP4Deserializer.m_RankMPI == 0 && !lasterrmsg.empty())
        {
            throw std::ios_base::failure("ERROR: File " + m_Name +
                                         " cannot be opened: " + lasterrmsg);
        }
        else
        {
            throw std::ios_base::failure("File " + m_Name +
                                         " cannot be opened");
        }
    }
    else if (flag == 1)
    {
        if (m_BP4Deserializer.m_RankMPI == 0)
        {
            throw std::ios_base::failure(
                "ERROR: File " + m_Name + " could not be found within the " +
                std::to_string(timeoutSeconds.count()) +
                "s timeout: " + lasterrmsg);
        }
        else
        {
            throw std::ios_base::failure(
                "ERROR: File " + m_Name + " could not be found within the " +
                std::to_string(timeoutSeconds.count()) + "s timeout");
        }
    }

    /* At this point we may have an empty index table.
     * The writer has created the file but no content may have been stored yet.
     */
}

void BP4Reader::InitTransports()
{
    if (m_IO.m_TransportsParameters.empty())
    {
        Params defaultTransportParameters;
        defaultTransportParameters["transport"] = "File";
        m_IO.m_TransportsParameters.push_back(defaultTransportParameters);
    }
}

/* Count index records to minimum 1 and maximum of N records so that
 * expected metadata size is less then a predetermined constant
 */
void MetadataCalculateMinFileSize(
    const format::BP4Deserializer &m_BP4Deserializer,
    const std::string &IdxFileName, char *buf, size_t idxsize, bool hasHeader,
    const size_t mdStartPos, size_t &newIdxSize, size_t &expectedMinFileSize)
{
    newIdxSize = 0;
    expectedMinFileSize = 0;

    if (hasHeader && idxsize < m_BP4Deserializer.m_IndexRecordSize)
    {
        return;
    }

    /* eliminate header for now for only calculating with records */
    if (hasHeader)
    {
        buf += m_BP4Deserializer.m_IndexRecordSize;
        idxsize -= m_BP4Deserializer.m_IndexRecordSize;
    }

    if (idxsize % m_BP4Deserializer.m_IndexRecordSize != 0)
    {
        throw std::runtime_error(
            "FATAL CODING ERROR: ADIOS Index file " + IdxFileName +
            " is assumed to always contain n*" +
            std::to_string(m_BP4Deserializer.m_IndexRecordSize) +
            " byte-length records. "
            "Right now the length of index buffer is " +
            std::to_string(idxsize) + " bytes.");
    }

    const int nTotalRecords = idxsize / m_BP4Deserializer.m_IndexRecordSize;
    if (nTotalRecords == 0)
    {
        // no (new) step entry in the index, so no metadata is expected
        newIdxSize = 0;
        expectedMinFileSize = 0;
        return;
    }

    int nRecords = 1;
    expectedMinFileSize = *(uint64_t *)&(
        buf[nRecords * m_BP4Deserializer.m_IndexRecordSize - 24]);
    while (nRecords < nTotalRecords)
    {
        const int n = nRecords + 1;
        const uint64_t mdEndPos =
            *(uint64_t *)&(buf[n * m_BP4Deserializer.m_IndexRecordSize - 24]);
        if (mdEndPos - mdStartPos > 16777216)
        {
            break;
        }
        expectedMinFileSize = mdEndPos;
        ++nRecords;
    }
    newIdxSize = nRecords * m_BP4Deserializer.m_IndexRecordSize;
    if (hasHeader)
    {
        newIdxSize += m_BP4Deserializer.m_IndexRecordSize;
    }
}

/* Count index records to minimum 1 and maximum of N records so that
 * expected metadata size is less then a predetermined constant
 */
void MetadataCalculateMinFileSize2(
    const format::BP4Deserializer &m_BP4Deserializer,
    const std::string &IdxFileName, char *buf, const size_t idxsize,
    size_t &newIdxSize, size_t &expectedMinFileSize)
{
    if (idxsize % m_BP4Deserializer.m_IndexRecordSize != 0)
    {
        throw std::runtime_error(
            "FATAL CODING ERROR: ADIOS Index file " + IdxFileName +
            " is assumed to always contain n*" +
            std::to_string(m_BP4Deserializer.m_IndexRecordSize) +
            " byte-length records. "
            "The file size now is " +
            std::to_string(idxsize + m_BP4Deserializer.m_IndexHeaderSize) +
            " bytes.");
    }

    const int nTotalRecords = idxsize / m_BP4Deserializer.m_IndexRecordSize;
    if (nTotalRecords == 0)
    {
        // no (new) step entry in the index, so no metadata is expected
        newIdxSize = 0;
        expectedMinFileSize = 0;
        return;
    }

    int nRecords = 1;
    expectedMinFileSize = *(uint64_t *)&(
        buf[nRecords * m_BP4Deserializer.m_IndexRecordSize - 24]);
    while (nRecords < nTotalRecords)
    {
        const int n = nRecords + 1;
        const uint64_t mdsize =
            *(uint64_t *)&(buf[n * m_BP4Deserializer.m_IndexRecordSize - 24]);
        if (mdsize > 16777216)
        {
            break;
        }
        expectedMinFileSize = mdsize;
        ++nRecords;
    }
    newIdxSize = nRecords * m_BP4Deserializer.m_IndexRecordSize;
}

uint64_t
MetadataExpectedMinFileSize(const format::BP4Deserializer &m_BP4Deserializer,
                            const std::string &IdxFileName, bool hasHeader)
{
    size_t idxsize = m_BP4Deserializer.m_MetadataIndex.m_Buffer.size();
    if (idxsize % 64 != 0)
    {
        throw std::runtime_error(
            "FATAL CODING ERROR: ADIOS Index file " + IdxFileName +
            " is assumed to always contain n*64 byte-length records. "
            "The file size now is " +
            std::to_string(idxsize) + " bytes.");
    }
    if ((hasHeader && idxsize < m_BP4Deserializer.m_IndexHeaderSize +
                                    m_BP4Deserializer.m_IndexRecordSize) ||
        idxsize < m_BP4Deserializer.m_IndexRecordSize)
    {
        // no (new) step entry in the index, so no metadata is expected
        return 0;
    }
    uint64_t lastpos = *(uint64_t *)&(
        m_BP4Deserializer.m_MetadataIndex.m_Buffer[idxsize - 24]);
    return lastpos;
}
<commit_msg>remove unused function and fix windows build issue<commit_after>/*
 * Distributed under the OSI-approved Apache License, Version 2.0.  See
 * accompanying file Copyright.txt for details.
 *
 * BP4Reader.cpp
 *
 *  Created on: Aug 1, 2018
 *      Author: <NAME> <EMAIL>
 */

#include "BP4Reader.h"
#include "BP4Reader.tcc"

#include "adios2/toolkit/profiling/taustubs/tautimer.hpp"

#include <chrono>
#include <errno.h>

namespace adios2
{
namespace core
{
namespace engine
{

BP4Reader::BP4Reader(IO &io, const std::string &name, const Mode mode,
                     helper::Comm comm)
: Engine("BP4Reader", io, name, mode, std::move(comm)),
  m_BP4Deserializer(m_Comm), m_MDFileManager(m_Comm), m_DataFileManager(m_Comm),
  m_MDIndexFileManager(m_Comm), m_ActiveFlagFileManager(m_Comm)
{
    TAU_SCOPED_TIMER("BP4Reader::Open");
    Init();
}

StepStatus BP4Reader::BeginStep(StepMode mode, const float timeoutSeconds)
{
    TAU_SCOPED_TIMER("BP4Reader::BeginStep");
    if (mode != StepMode::Read)
    {
        throw std::invalid_argument("ERROR: mode is not supported yet, "
                                    "only Read is valid for "
                                    "engine BP4Reader, in call to "
                                    "BeginStep\n");
    }

    if (!m_BP4Deserializer.m_DeferredVariables.empty())
    {
        throw std::invalid_argument(
            "ERROR: existing variables subscribed with "
            "GetDeferred, did you forget to call "
            "PerformGets() or EndStep()?, in call to BeginStep\n");
    }

    // used to inquire for variables in streaming mode
    m_IO.m_ReadStreaming = true;
    StepStatus status = StepStatus::OK;

    if (m_FirstStep)
    {
        if (m_BP4Deserializer.m_MetadataSet.StepsCount == 0)
        {
            status = CheckForNewSteps(Seconds(timeoutSeconds));
        }
    }
    else
    {
        if (m_CurrentStep + 1 >= m_BP4Deserializer.m_MetadataSet.StepsCount)
        {
            status = CheckForNewSteps(Seconds(timeoutSeconds));
        }
    }

    // This should be after getting new steps

    if (status == StepStatus::OK)
    {
        if (m_FirstStep)
        {
            m_FirstStep = false;
        }
        else
        {
            ++m_CurrentStep;
        }

        m_IO.m_EngineStep = m_CurrentStep;
        m_IO.ResetVariablesStepSelection(false,
                                         "in call to BP4 Reader BeginStep");

        // caches attributes for each step
        // if a variable name is a prefix
        // e.g. var  prefix = {var/v1, var/v2, var/v3}
        m_IO.SetPrefixedNames(true);
    }

    return status;
}

size_t BP4Reader::CurrentStep() const { return m_CurrentStep; }

void BP4Reader::EndStep()
{
    TAU_SCOPED_TIMER("BP4Reader::EndStep");
    PerformGets();
}

void BP4Reader::PerformGets()
{
    TAU_SCOPED_TIMER("BP4Reader::PerformGets");
    if (m_BP4Deserializer.m_DeferredVariables.empty())
    {
        return;
    }

    for (const std::string &name : m_BP4Deserializer.m_DeferredVariables)
    {
        const DataType type = m_IO.InquireVariableType(name);

        if (type == DataType::Compound)
        {
        }
#define declare_type(T)                                                        \
    else if (type == helper::GetDataType<T>())                                 \
    {                                                                          \
        Variable<T> &variable =                                                \
            FindVariable<T>(name, "in call to PerformGets, EndStep or Close"); \
        for (auto &blockInfo : variable.m_BlocksInfo)                          \
        {                                                                      \
            m_BP4Deserializer.SetVariableBlockInfo(variable, blockInfo);       \
        }                                                                      \
        ReadVariableBlocks(variable);                                          \
        variable.m_BlocksInfo.clear();                                         \
    }
        ADIOS2_FOREACH_STDTYPE_1ARG(declare_type)
#undef declare_type
    }

    m_BP4Deserializer.m_DeferredVariables.clear();
}

// PRIVATE
void BP4Reader::Init()
{
    if (m_OpenMode != Mode::Read)
    {
        throw std::invalid_argument("ERROR: BPFileReader only "
                                    "supports OpenMode::Read from" +
                                    m_Name + " " + m_EndMessage);
    }

    m_BP4Deserializer.Init(m_IO.m_Parameters, "in call to BP4::Open to write");
    InitTransports();

    /* Do a collective wait for the file(s) to appear within timeout.
       Make sure every process comes to the same conclusion */
    const Seconds timeoutSeconds =
        Seconds(m_BP4Deserializer.m_Parameters.OpenTimeoutSecs);

    // set poll to 1/100 of timeout
    Seconds pollSeconds = timeoutSeconds / 100.0;
    static const auto pollSecondsMin = Seconds(1.0);
    if (pollSeconds < pollSecondsMin)
    {
        pollSeconds = pollSecondsMin;
    }
    static const auto pollSecondsMax = Seconds(10.0);
    if (pollSeconds > pollSecondsMax)
    {
        pollSeconds = pollSecondsMax;
    }

    TimePoint timeoutInstant =
        std::chrono::steady_clock::now() + timeoutSeconds;

    OpenFiles(timeoutInstant, pollSeconds, timeoutSeconds);
    if (!m_BP4Deserializer.m_Parameters.StreamReader)
    {
        /* non-stream reader gets as much steps as available now */
        InitBuffer(timeoutInstant, pollSeconds / 10, timeoutSeconds);
    }
}

bool BP4Reader::SleepOrQuit(const TimePoint &timeoutInstant,
                            const Seconds &pollSeconds)
{
    auto now = std::chrono::steady_clock::now();
    if (now + pollSeconds >= timeoutInstant)
    {
        return false;
    }
    auto remainderTime = timeoutInstant - now;
    auto sleepTime = pollSeconds;
    if (remainderTime < sleepTime)
    {
        sleepTime = remainderTime;
    }
    std::this_thread::sleep_for(sleepTime);
    return true;
}

size_t BP4Reader::OpenWithTimeout(transportman::TransportMan &tm,
                                  const std::vector<std::string> &fileNames,
                                  const TimePoint &timeoutInstant,
                                  const Seconds &pollSeconds,
                                  std::string &lasterrmsg /*INOUT*/)
{
    size_t flag = 1; // 0 = OK, opened file, 1 = timeout, 2 = error
    do
    {
        try
        {
            errno = 0;
            const bool profile = m_BP4Deserializer.m_Profiler.m_IsActive;
            tm.OpenFiles(fileNames, adios2::Mode::Read,
                         m_IO.m_TransportsParameters, profile);
            flag = 0; // found file
            break;
        }
        catch (std::ios_base::failure &e)
        {
            lasterrmsg =
                std::string("errno=" + std::to_string(errno) + ": " + e.what());
            if (errno == ENOENT)
            {
                flag = 1; // timeout
            }
            else
            {
                flag = 2; // fatal error
                break;
            }
        }
    } while (SleepOrQuit(timeoutInstant, pollSeconds));
    return flag;
}

void BP4Reader::OpenFiles(TimePoint &timeoutInstant, const Seconds &pollSeconds,
                          const Seconds &timeoutSeconds)
{
    /* Poll */
    size_t flag = 1; // 0 = OK, opened file, 1 = timeout, 2 = error
    std::string lasterrmsg;
    if (m_BP4Deserializer.m_RankMPI == 0)
    {
        /* Open the metadata index table */
        const std::string metadataIndexFile(
            m_BP4Deserializer.GetBPMetadataIndexFileName(m_Name));

        flag = OpenWithTimeout(m_MDIndexFileManager, {metadataIndexFile},
                               timeoutInstant, pollSeconds, lasterrmsg);
        if (flag == 0)
        {
            /* Open the metadata file */
            const std::string metadataFile(
                m_BP4Deserializer.GetBPMetadataFileName(m_Name));

            /* We found md.idx. If we don't find md.0 immediately  we should
             * wait a little bit hoping for the file system to catch up.
             * This slows down finding the error in file reading mode but
             * it will be more robust in streaming mode
             */
            if (timeoutSeconds == Seconds(0.0))
            {
                timeoutInstant += Seconds(5.0);
            }

            flag = OpenWithTimeout(m_MDFileManager, {metadataFile},
                                   timeoutInstant, pollSeconds, lasterrmsg);
            if (flag != 0)
            {
                /* Close the metadata index table */
                m_MDIndexFileManager.CloseFiles();
            }
        }
    }

    flag = m_Comm.BroadcastValue(flag, 0);
    if (flag == 2)
    {
        if (m_BP4Deserializer.m_RankMPI == 0 && !lasterrmsg.empty())
        {
            throw std::ios_base::failure("ERROR: File " + m_Name +
                                         " cannot be opened: " + lasterrmsg);
        }
        else
        {
            throw std::ios_base::failure("File " + m_Name +
                                         " cannot be opened");
        }
    }
    else if (flag == 1)
    {
        if (m_BP4Deserializer.m_RankMPI == 0)
        {
            throw std::ios_base::failure(
                "ERROR: File " + m_Name + " could not be found within the " +
                std::to_string(timeoutSeconds.count()) +
                "s timeout: " + lasterrmsg);
        }
        else
        {
            throw std::ios_base::failure(
                "ERROR: File " + m_Name + " could not be found within the " +
                std::to_string(timeoutSeconds.count()) + "s timeout");
        }
    }

    /* At this point we may have an empty index table.
     * The writer has created the file but no content may have been stored yet.
     */
}

void BP4Reader::InitTransports()
{
    if (m_IO.m_TransportsParameters.empty())
    {
        Params defaultTransportParameters;
        defaultTransportParameters["transport"] = "File";
        m_IO.m_TransportsParameters.push_back(defaultTransportParameters);
    }
}

/* Count index records to minimum 1 and maximum of N records so that
 * expected metadata size is less then a predetermined constant
 */
void MetadataCalculateMinFileSize(
    const format::BP4Deserializer &m_BP4Deserializer,
    const std::string &IdxFileName, char *buf, size_t idxsize, bool hasHeader,
    const size_t mdStartPos, size_t &newIdxSize, size_t &expectedMinFileSize)
{
    newIdxSize = 0;
    expectedMinFileSize = 0;

    if (hasHeader && idxsize < m_BP4Deserializer.m_IndexRecordSize)
    {
        return;
    }

    /* eliminate header for now for only calculating with records */
    if (hasHeader)
    {
        buf += m_BP4Deserializer.m_IndexRecordSize;
        idxsize -= m_BP4Deserializer.m_IndexRecordSize;
    }

    if (idxsize % m_BP4Deserializer.m_IndexRecordSize != 0)
    {
        throw std::runtime_error(
            "FATAL CODING ERROR: ADIOS Index file " + IdxFileName +
            " is assumed to always contain n*" +
            std::to_string(m_BP4Deserializer.m_IndexRecordSize) +
            " byte-length records. "
            "Right now the length of index buffer is " +
            std::to_string(idxsize) + " bytes.");
    }

    const size_t nTotalRecords = idxsize / m_BP4Deserializer.m_IndexRecordSize;
    if (nTotalRecords == 0)
    {
        // no (new) step entry in the index, so no metadata is expected
        newIdxSize = 0;
        expectedMinFileSize = 0;
        return;
    }

    size_t nRecords = 1;
    expectedMinFileSize = *(uint64_t *)&(
        buf[nRecords * m_BP4Deserializer.m_IndexRecordSize - 24]);
    while (nRecords < nTotalRecords)
    {
        const size_t n = nRecords + 1;
        const uint64_t mdEndPos =
            *(uint64_t *)&(buf[n * m_BP4Deserializer.m_IndexRecordSize - 24]);
        if (mdEndPos - mdStartPos > 16777216)
        {
            break;
        }
        expectedMinFileSize = mdEndPos;
        ++nRecords;
    }
    newIdxSize = nRecords * m_BP4Deserializer.m_IndexRecordSize;
    if (hasHeader)
    {
        newIdxSize += m_BP4Deserializer.m_IndexRecordSize;
    }
}

uint64_t
MetadataExpectedMinFileSize(const format::BP4Deserializer &m_BP4Deserializer,
                            const std::string &IdxFileName, bool hasHeader)
{
    size_t idxsize = m_BP4Deserializer.m_MetadataIndex.m_Buffer.size();
    if (idxsize % 64 != 0)
    {
        throw std::runtime_error(
            "FATAL CODING ERROR: ADIOS Index file " + IdxFileName +
            " is assumed to always contain n*64 byte-length records. "
            "The file size now is " +
            std::to_string(idxsize) + " bytes.");
    }
    if ((hasHeader && idxsize < m_BP4Deserializer.m_IndexHeaderSize +
                                    m_BP4Deserializer.m_IndexRecordSize) ||
        idxsize < m_BP4Deserializer.m_IndexRecordSize)
    {
        // no (new) step entry in the index, so no metadata is expected
        return 0;
    }
    uint64_t lastpos = *(uint64_t *)&(
        m_BP4Deserializer.m_MetadataIndex.m_Buffer[idxsize - 24]);
    return lastpos;
}

{'fraction_non_alphanumeric': 0.09436458556818994, 'fraction_numerical': 0.012407923907602088, 'mean_word_length': 1.8439088875330487, 'pattern_counts': {'":': 2, '<': 32, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 64, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6598516', 'n_tokens_mistral': 8390, 'n_tokens_neox': 8052, 'n_words': 1890}