<commit_before>#include "BFSEntryWriter.hpp"
#include "Detail.hpp"

#include <boost/make_shared.hpp>

namespace bfs
{

    namespace detail
    {
        /**
         * @brief gets number of file blocks required to store file
         * @param fsize the size of the file to store
         * @return the number of file blocks required
         */
        inline uint64_t computeBlocksRequired(uint64_t const fsize)
        {
            // file blocks to also store filename which also needs to be
            // taken in to account
            uint64_t const fSizePlusFileNameLength = fsize + MAX_FILENAME_LENGTH;
            // 20:
            // 4 for number of bytes utilized in a given block
            // 8 for next block index
            // 8 for previous block index
            uint64_t const blockSizeWithoutMetaStuff = FILE_BLOCK_SIZE - 20;

            // compute how many 512 byte blocks are required to store the file
            uint64_t blocksRequired(0);
            if (fSizePlusFileNameLength < blockSizeWithoutMetaStuff) {
                ++blocksRequired; // 1 block required
                return blocksRequired;
            } else {
                uint64_t const leftOver = fSizePlusFileNameLength % blockSizeWithoutMetaStuff;
                uint64_t const roundedDown = fSizePlusFileNameLength - leftOver;
                uint64_t const basalBlocks = roundedDown / blockSizeWithoutMetaStuff;
                blocksRequired = basalBlocks;
                if (leftOver > 0) {
                    ++blocksRequired;
                }
                return blocksRequired;
            }
        }
    }

    BFSEntryWriter::BFSEntryWriter(std::string const &bfsOutputPath,
                                   std::string const &entryName,
                                   uint64_t const fsize,
                                   uint64_t const parentIndex)
        : m_bfsOutputPath(bfsOutputPath)
        , m_entryName(entryName)
        , m_fsize(fsize)
        , m_parentIndex(parentIndex)
        , m_bfsOutputStream(boost::make_shared<std::fstream>(bfsOutputPath.c_str(), std::ios::in | std::ios::out | std::ios::binary))
        , m_totalBlocks(detail::getBlockCount(*m_bfsOutputStream))
        , m_blocksToUse(detail::getNAvailableBlocks(*m_bfsOutputStream, detail::computeBlocksRequired(fsize), m_totalBlocks))
          //, m_metaOffset(detail::getOffsetOfNextFreeMetaSpaceBlock(*m_bfsOutputStream))
          //   , m_fileOffset(detail::getOffsetOfNextFreeFileSpaceBlock(*m_bfsOutputStream))
    {
        //updateSuperBlock();
        //writeMetaBlock();
        //writeFileName();
    }


    /**
     * writes the file metadata which consists of:-
     *
     * 8 bytes specifying the size
     * 8 bytes specifying the offset in the bfs image
     * 8 bytes specifying the parent folder index
     * 8 bytes specifying other data (e.g. read-only, last modified etc)
     */
    void
    BFSEntryWriter::writeMetaBlock()
    {
        uint8_t sizeBytes[8];
        detail::convertInt64ToInt8Array(m_fsize, sizeBytes);
        uint8_t posBytes[8];
        detail::convertInt64ToInt8Array(m_fileOffset, posBytes);
        uint8_t parentBytes[8];
        detail::convertInt64ToInt8Array(m_parentIndex, parentBytes);
        (void)m_bfsOutputStream->seekp((std::streampos)m_metaOffset);
        m_bfsOutputStream->write((char*)sizeBytes, 8);    // size of file
        m_bfsOutputStream->write((char*)posBytes, 8);     // position in main file space
        m_bfsOutputStream->write((char*)parentBytes, 8);  // index of parent directory
        m_bfsOutputStream->write((char*)parentBytes, 8);  // extra space for other stuff (tbd)
    }

    /**
     * Updates file count and total file size information in the superblock
     */
    void BFSEntryWriter::updateSuperBlock()
    {
        detail::incrementFileCount(*m_bfsOutputStream);
        //detail::updateFileSpaceAccumulator(*m_bfsOutputStream, m_fsize);
    }


    void
    BFSEntryWriter::bufferBytesUsedForFileBlockN()
    {
        uint64_t const bytes = m_totalBlocks / uint64_t(8);
        uint64_t const offset = detail::getOffsetOfFileBlock(m_blocksToUse[m_currentBlockIndex], m_totalBlocks);

        // file blocks to also store filename which also needs to be
        // taken in to account
        uint64_t const fSizePlusFileNameLength = m_fsize + detail::MAX_FILENAME_LENGTH;
        // 20:
        // 4 for number of bytes utilized in a given block
        // 8 for next block index
        // 8 for previous block index
        uint64_t const blockSizeWithoutMetaStuff = detail::FILE_BLOCK_SIZE - 20;
        if (fSizePlusFileNameLength < blockSizeWithoutMetaStuff) {
            uint8_t dat[4];
            detail::convertInt32ToInt4Array((uint32_t)fSizePlusFileNameLength, dat);
            for(int i = 0; i < 4; ++i) {m_dataBuffer.push_back(dat[i]);}
        } else {
            uint64_t const leftOver = fSizePlusFileNameLength % blockSizeWithoutMetaStuff;
            uint64_t const roundedDown = fSizePlusFileNameLength - leftOver;
            uint64_t const basalBlocks = roundedDown / blockSizeWithoutMetaStuff;
            uint8_t dat[4];
            if (m_currentBlockIndex < basalBlocks) {
                detail::convertInt32ToInt4Array((uint32_t)blockSizeWithoutMetaStuff, dat);
                for(int i = 0; i < 4; ++i) {m_dataBuffer.push_back(dat[i]);}
            } else {
                detail::convertInt32ToInt4Array((uint32_t)leftOver, dat);
                for(int i = 0; i < 4; ++i) {m_dataBuffer.push_back(dat[i]);}
            }
        }
    }

    void
    BFSEntryWriter::bufferLastAndNextBlockIndicesForFileBlockN(uint64_t const lastBlockIndex,
                                                              uint64_t const nextBlockIndex)
    {
        uint64_t const bytes = m_totalBlocks / uint64_t(8);
        // note, the '4' on the end signifies skip past the 4 byte 'bytes used' block
        uint64_t const offset = detail::getOffsetOfFileBlock(m_blocksToUse[m_currentBlockIndex], m_totalBlocks) + 4;
        uint8_t dat[8];
        detail::convertInt64ToInt8Array(m_blocksToUse[lastBlockIndex], dat);
        for(int i = 0; i < 8; ++i) {m_dataBuffer.push_back(dat[i]);}
        detail::convertInt64ToInt8Array(m_blocksToUse[nextBlockIndex], dat);
        for(int i = 0; i < 8; ++i) {m_dataBuffer.push_back(dat[i]);}
    }

    void
    BFSEntryWriter::bufferByte(char const byte)
    {
        m_dataBuffer.push_back(byte);
        ++m_buffered;
        if(m_buffered == m_fsize + detail::MAX_FILENAME_LENGTH) {

            // write out remaining bytes
            uint64_t const bytes = m_totalBlocks / uint64_t(8);
            uint64_t const offset = detail::getOffsetOfFileBlock(m_blocksToUse[m_currentBlockIndex], m_totalBlocks);
            (void)m_bfsOutputStream->seekp(offset);
            (void)m_bfsOutputStream->write((char*)&m_dataBuffer.front(), m_dataBuffer.size());
            m_dataBuffer.clear();

        } else {
            if(m_dataBuffer.size() == detail::FILE_BLOCK_SIZE) {

                // write
                // clear
                // update burrent block index
                // start again

                uint64_t const bytes = m_totalBlocks / uint64_t(8);
                uint64_t const offset = detail::getOffsetOfFileBlock(m_blocksToUse[m_currentBlockIndex], m_totalBlocks);
                (void)m_bfsOutputStream->seekp(offset);
                (void)m_bfsOutputStream->write((char*)&m_dataBuffer.front(), m_dataBuffer.size());

                m_dataBuffer.clear();
                ++m_currentBlockIndex;

            } else {
                // write how many bytes will be occupied
                uint64_t bufferSize = computeBufferSize(m_currentBlockIndex);
                bufferBytesUsedForFileBlockN();
                uint64_t prev;
                uint64_t next;
                computePreviousAndNextBlockIndices(prev, next, m_currentBlockIndex);
                bufferLastAndNextBlockIndicesForFileBlockN(prev, next);
            }
        }
    }
<commit_msg>bug fix in sink<commit_after>#include "BFSEntryWriter.hpp"
#include "Detail.hpp"

#include <boost/make_shared.hpp>

namespace bfs
{

    namespace detail
    {
        /**
         * @brief gets number of file blocks required to store file
         * @param fsize the size of the file to store
         * @return the number of file blocks required
         */
        inline uint64_t computeBlocksRequired(uint64_t const fsize)
        {
            // file blocks to also store filename which also needs to be
            // taken in to account
            uint64_t const fSizePlusFileNameLength = fsize + MAX_FILENAME_LENGTH;
            // 20:
            // 4 for number of bytes utilized in a given block
            // 8 for next block index
            // 8 for previous block index
            uint64_t const blockSizeWithoutMetaStuff = FILE_BLOCK_SIZE - 20;

            // compute how many 512 byte blocks are required to store the file
            uint64_t blocksRequired(0);
            if (fSizePlusFileNameLength < blockSizeWithoutMetaStuff) {
                ++blocksRequired; // 1 block required
                return blocksRequired;
            } else {
                uint64_t const leftOver = fSizePlusFileNameLength % blockSizeWithoutMetaStuff;
                uint64_t const roundedDown = fSizePlusFileNameLength - leftOver;
                uint64_t const basalBlocks = roundedDown / blockSizeWithoutMetaStuff;
                blocksRequired = basalBlocks;
                if (leftOver > 0) {
                    ++blocksRequired;
                }
                return blocksRequired;
            }
        }
    }

    BFSEntryWriter::BFSEntryWriter(std::string const &bfsOutputPath,
                                   std::string const &entryName,
                                   uint64_t const fsize,
                                   uint64_t const parentIndex)
        : m_bfsOutputPath(bfsOutputPath)
        , m_entryName(entryName)
        , m_fsize(fsize)
        , m_parentIndex(parentIndex)
        , m_bfsOutputStream(boost::make_shared<std::fstream>(bfsOutputPath.c_str(), std::ios::in | std::ios::out | std::ios::binary))
        , m_totalBlocks(detail::getBlockCount(*m_bfsOutputStream))
        , m_blocksToUse(detail::getNAvailableBlocks(*m_bfsOutputStream, detail::computeBlocksRequired(fsize), m_totalBlocks))
          //, m_metaOffset(detail::getOffsetOfNextFreeMetaSpaceBlock(*m_bfsOutputStream))
    {
        //updateSuperBlock();
        //writeMetaBlock();
        //writeFileName();
    }


    /**
     * writes the file metadata which consists of:-
     *
     * 8 bytes specifying the size
     * 8 bytes specifying the offset in the bfs image
     * 8 bytes specifying the parent folder index
     * 8 bytes specifying other data (e.g. read-only, last modified etc)
     */
    void
    BFSEntryWriter::writeMetaBlock()
    {
        uint8_t sizeBytes[8];
        detail::convertInt64ToInt8Array(m_fsize, sizeBytes);
        uint8_t posBytes[8];
        detail::convertInt64ToInt8Array(m_fileOffset, posBytes);
        uint8_t parentBytes[8];
        detail::convertInt64ToInt8Array(m_parentIndex, parentBytes);
        (void)m_bfsOutputStream->seekp((std::streampos)m_metaOffset);
        m_bfsOutputStream->write((char*)sizeBytes, 8);    // size of file
        m_bfsOutputStream->write((char*)posBytes, 8);     // position in main file space
        m_bfsOutputStream->write((char*)parentBytes, 8);  // index of parent directory
        m_bfsOutputStream->write((char*)parentBytes, 8);  // extra space for other stuff (tbd)
    }

    /**
     * Updates file count and total file size information in the superblock
     */
    void BFSEntryWriter::updateSuperBlock()
    {
        detail::incrementFileCount(*m_bfsOutputStream);
        //detail::updateFileSpaceAccumulator(*m_bfsOutputStream, m_fsize);
    }


    void
    BFSEntryWriter::bufferBytesUsedForFileBlockN()
    {
        uint64_t const bytes = m_totalBlocks / uint64_t(8);
        uint64_t const offset = detail::getOffsetOfFileBlock(m_blocksToUse[m_currentBlockIndex], m_totalBlocks);

        // file blocks to also store filename which also needs to be
        // taken in to account
        uint64_t const fSizePlusFileNameLength = m_fsize + detail::MAX_FILENAME_LENGTH;
        // 20:
        // 4 for number of bytes utilized in a given block
        // 8 for next block index
        // 8 for previous block index
        uint64_t const blockSizeWithoutMetaStuff = detail::FILE_BLOCK_SIZE - 20;
        if (fSizePlusFileNameLength < blockSizeWithoutMetaStuff) {
            uint8_t dat[4];
            detail::convertInt32ToInt4Array((uint32_t)fSizePlusFileNameLength, dat);
            for(int i = 0; i < 4; ++i) {m_dataBuffer.push_back(dat[i]);}
        } else {
            uint64_t const leftOver = fSizePlusFileNameLength % blockSizeWithoutMetaStuff;
            uint64_t const roundedDown = fSizePlusFileNameLength - leftOver;
            uint64_t const basalBlocks = roundedDown / blockSizeWithoutMetaStuff;
            uint8_t dat[4];
            if (m_currentBlockIndex < basalBlocks) {
                detail::convertInt32ToInt4Array((uint32_t)blockSizeWithoutMetaStuff, dat);
                for(int i = 0; i < 4; ++i) {m_dataBuffer.push_back(dat[i]);}
            } else {
                detail::convertInt32ToInt4Array((uint32_t)leftOver, dat);
                for(int i = 0; i < 4; ++i) {m_dataBuffer.push_back(dat[i]);}
            }
        }
    }

    void
    BFSEntryWriter::bufferLastAndNextBlockIndicesForFileBlockN(uint64_t const lastBlockIndex,
                                                              uint64_t const nextBlockIndex)
    {
        uint64_t const bytes = m_totalBlocks / uint64_t(8);
        // note, the '4' on the end signifies skip past the 4 byte 'bytes used' block
        uint64_t const offset = detail::getOffsetOfFileBlock(m_blocksToUse[m_currentBlockIndex], m_totalBlocks) + 4;
        uint8_t dat[8];
        detail::convertInt64ToInt8Array(m_blocksToUse[lastBlockIndex], dat);
        for(int i = 0; i < 8; ++i) {m_dataBuffer.push_back(dat[i]);}
        detail::convertInt64ToInt8Array(m_blocksToUse[nextBlockIndex], dat);
        for(int i = 0; i < 8; ++i) {m_dataBuffer.push_back(dat[i]);}
    }

    void
    BFSEntryWriter::bufferByte(char const byte)
    {
        m_dataBuffer.push_back(byte);
        ++m_buffered;
        if(m_buffered == m_fsize + detail::MAX_FILENAME_LENGTH) {

            // write out remaining bytes
            uint64_t const bytes = m_totalBlocks / uint64_t(8);
            uint64_t const offset = detail::getOffsetOfFileBlock(m_blocksToUse[m_currentBlockIndex], m_totalBlocks);
            (void)m_bfsOutputStream->seekp(offset);
            (void)m_bfsOutputStream->write((char*)&m_dataBuffer.front(), m_dataBuffer.size());
            m_dataBuffer.clear();

        } else {
            if(m_dataBuffer.size() == detail::FILE_BLOCK_SIZE) {

                // write
                // clear
                // update current block index
                // start again

                uint64_t const bytes = m_totalBlocks / uint64_t(8);
                uint64_t const offset = detail::getOffsetOfFileBlock(m_blocksToUse[m_currentBlockIndex], m_totalBlocks);
                (void)m_bfsOutputStream->seekp(offset);
                (void)m_bfsOutputStream->write((char*)&m_dataBuffer.front(), m_dataBuffer.size());

                m_dataBuffer.clear();
                ++m_currentBlockIndex;

                // write how many bytes will be occupied in the next block
                uint64_t bufferSize = computeBufferSize(m_currentBlockIndex);
                bufferBytesUsedForFileBlockN();
                uint64_t prev;
                uint64_t next;
                computePreviousAndNextBlockIndices(prev, next, m_currentBlockIndex);
                bufferLastAndNextBlockIndicesForFileBlockN(prev, next);
            }
        }
    }

{'fraction_non_alphanumeric': 0.08907185628742514, 'fraction_numerical': 0.0188373253493014, 'mean_word_length': 2.2693719412724307, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '610708', 'n_tokens_mistral': 4777, 'n_tokens_neox': 4476, 'n_words': 999}