<commit_before>/**
 * @file dem_stream_file.cpp
 * @author <NAME> <me (at) invokr (dot) org>
 * @version 1.0
 *
 * @par License
 *    Alice Replay Parser
 *    Copyright 2014 <NAME>
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */


#include <snappy.h>

#include <alice/demo.pb.h>
#include <alice/dem_stream_file.hpp>

namespace dota {
    void dem_stream_file::open(std::string path) {
        // open stream
        stream.open(path.c_str(), std::ifstream::in | std::ifstream::binary);

        // check if it was successful
        if (!stream.is_open())
            BOOST_THROW_EXCEPTION(demFileNotAccessible()
                << EArg<1>::info(path)
            );

        // check filesize
        const std::streampos fstart = stream.tellg();
        stream.seekg (0, std::ios::end);
        std::streampos fsize = stream.tellg() - fstart;
        stream.seekg(fstart);

        if (fsize < sizeof(demHeader_t))
            BOOST_THROW_EXCEPTION((demFileTooSmall()
                << EArg<1>::info(path)
                << EArgT<2, std::streampos>::info(fsize)
                << EArgT<3, std::size_t>::info(sizeof(demHeader_t))
            ));

        // verify the header
        demHeader_t head;
        stream.read((char*) &head, sizeof(demHeader_t));
        if(strcmp(head.headerid, DOTA_DEMHEADERID))
            BOOST_THROW_EXCEPTION(demHeaderMismatch()
                << EArg<1>::info(path)
                << EArg<2>::info(std::string(head.headerid, 8))
                << EArg<3>::info(std::string(DOTA_DEMHEADERID))
            );

        // save path for later
        file = path;
    }

    demMessage_t dem_stream_file::read(bool skip) {
        // Make sure the buffers has been allocated
        assert(buffer != nullptr);
        assert(bufferSnappy != nullptr);

        // Get type / tick / size
        uint32_t type = readVarInt();
        const bool compressed = type & DEM_IsCompressed;
        type = (type & ~DEM_IsCompressed);

        uint32_t tick = readVarInt();
        uint32_t size = readVarInt();

        // Check if this is the last message
        if (parsingState == 1) parsingState = 2;
        if (type == 0)         parsingState = 1; // marks the message before the laste one

        // skip messages if skip is set
        if (skip) {
            stream.seekg(size, std::ios::cur); // seek forward
            return demMessage_t{false, 0, 0, nullptr, 0}; // return empty msg
        }

        // read into char array and convert to string
        if (size > DOTA_DEM_BUFSIZE)
            BOOST_THROW_EXCEPTION((demMessageToBig()
                << EArgT<1, std::size_t>::info(size)
            ));<commit_msg>Skipping is now equivalent to not-forwarding certain entries<commit_after>/**
 * @file dem_stream_file.cpp
 * @author <NAME> <me (at) invokr (dot) org>
 * @version 1.0
 *
 * @par License
 *    Alice Replay Parser
 *    Copyright 2014 <NAME>
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */


#include <set>
#include <snappy.h>

#include <alice/demo.pb.h>
#include <alice/dem_stream_file.hpp>

namespace dota {
    void dem_stream_file::open(std::string path) {
        // open stream
        stream.open(path.c_str(), std::ifstream::in | std::ifstream::binary);

        // check if it was successful
        if (!stream.is_open())
            BOOST_THROW_EXCEPTION(demFileNotAccessible()
                << EArg<1>::info(path)
            );

        // check filesize
        const std::streampos fstart = stream.tellg();
        stream.seekg (0, std::ios::end);
        std::streampos fsize = stream.tellg() - fstart;
        stream.seekg(fstart);

        if (fsize < sizeof(demHeader_t))
            BOOST_THROW_EXCEPTION((demFileTooSmall()
                << EArg<1>::info(path)
                << EArgT<2, std::streampos>::info(fsize)
                << EArgT<3, std::size_t>::info(sizeof(demHeader_t))
            ));

        // verify the header
        demHeader_t head;
        stream.read((char*) &head, sizeof(demHeader_t));
        if(strcmp(head.headerid, DOTA_DEMHEADERID))
            BOOST_THROW_EXCEPTION(demHeaderMismatch()
                << EArg<1>::info(path)
                << EArg<2>::info(std::string(head.headerid, 8))
                << EArg<3>::info(std::string(DOTA_DEMHEADERID))
            );

        // save path for later
        file = path;
    }

    demMessage_t dem_stream_file::read(bool skip) {
        // Make sure the buffers has been allocated
        assert(buffer != nullptr);
        assert(bufferSnappy != nullptr);

        // Static default list of packages that are not parsed
        static std::set<uint32_t> skips {
            1, 2, 3, 9, 10, 11, 12, 13, 14
        };

        // Get type / tick / size
        uint32_t type = readVarInt();
        const bool compressed = type & DEM_IsCompressed;
        type = (type & ~DEM_IsCompressed);

        uint32_t tick = readVarInt();
        uint32_t size = readVarInt();

        // Check if this is the last message
        if (parsingState == 1) parsingState = 2;
        if (type == 0)         parsingState = 1; // marks the message before the laste one

        // skip messages if skip is set
        if (skip && skips.count(type)) {
            stream.seekg(size, std::ios::cur); // seek forward
            return demMessage_t{false, 0, 0, nullptr, 0}; // return empty msg
        }

        // read into char array and convert to string
        if (size > DOTA_DEM_BUFSIZE)
            BOOST_THROW_EXCEPTION((demMessageToBig()
                << EArgT<1, std::size_t>::info(size)
            ));
{'fraction_non_alphanumeric': 0.11674439596810592, 'fraction_numerical': 0.012336392357454491, 'mean_word_length': 2.216255442670537, 'pattern_counts': {'":': 0, '<': 67, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6175437', 'n_tokens_mistral': 2072, 'n_tokens_neox': 1956, 'n_words': 553}