<commit_before>/**
 * @file parser.cpp
 * @author <NAME> <me (at) invokr (dot) org>
 * @version 1.0
 *
 * @par License
 *    Alice Replay Parser
 *    Copyright 2014 <NAME>
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

#include <alice/demo.pb.h>
#include <alice/netmessages.pb.h>
#include <alice/usermessages.pb.h>
#include <alice/dota_usermessages.pb.h>

#include <alice/bitstream.hpp>
#include <alice/parser.hpp>

namespace dota {
    parser::parser(const settings s, dem_stream *stream) : set(s), stream(stream), tick(0), msgs(0), sendtableId(-1),
        stringtableId(-1), delta(nullptr)
    {
        handlerRegisterCallback((&handler), msgDem, DEM_Packet,       parser, handlePacket)
        handlerRegisterCallback((&handler), msgDem, DEM_SignonPacket, parser, handlePacket)
        handlerRegisterCallback((&handler), msgNet, svc_UserMessage,  parser, handleUserMessage)

        if (set.parse_entities) {
            handlerRegisterCallback((&handler),  msgDem, DEM_ClassInfo,  parser, handleClasses)
            handlerRegisterCallback((&handler),  msgDem, DEM_SendTables, parser, handleSendTables)

            handlerRegisterCallback((&handler), msgNet, svc_ServerInfo, parser, handleServerInfo)
            handlerRegisterCallback((&handler), msgNet, svc_SendTable,  parser, handleSendTable)
        }

        if (set.parse_stringtables) {
            handlerRegisterCallback((&handler), msgNet, svc_CreateStringTable, parser, handleCreateStringtable)
            handlerRegisterCallback((&handler), msgNet, svc_UpdateStringTable, parser, handleUpdateStringtable)
        }

        if (set.parse_entities) {
            // allocate memory for entities
            entities.resize(DOTA_MAX_ENTITIES);

            // callback handlers
            handlerRegisterCallback((&handler), msgNet, svc_PacketEntities, parser, handleEntity)
        }

        if (set.track_entities) {
            delta = new entity_delta;
        }

        // Get unique / non-unique IDs for all messages
        registerTypes();
    }

    parser::~parser() {
        // free stream memory
        delete stream;

        // free sendprop table
        for (auto &tbl : sendtables) {
            tbl.value.free();
        }

        // free entity delta
        if (set.track_entities)
            delete delta;
    }

    void parser::open(std::string path) {
        file = path;
        stream->open(path);

        // let handlers know we begin to parse
        handler.forward<msgStatus>(REPLAY_START, REPLAY_START, 0);
    }

    bool parser::good() {
        return stream->good();
    }

    void parser::read() {
        // Read a single message
        //
        // If forward_dem is set, packages not handled by the parser are skipped because no
        // one would get them anyway.
        // This increases the read performance by about 20% for the DEM part.
        demMessage_t msg = stream->read(!set.forward_dem);
        ++msgs;

        // update current tick
        if (msg.tick > 0) // last ticks are 0ed
            tick = msg.tick;

        // Forward messages via the handler or handle them internaly
        if (set.forward_dem) {
            handler.forward<msgDem>(msg.type, std::move(msg), msg.tick);
        } else {
            #ifndef _MSC_VER
            switch (msg.type) {
                case DEM_ClassInfo: {
                    if (set.parse_entities) {
                        auto cb = handler.retrieve<msgDem::id>(msg.type, std::move(msg), msg.tick);
                        handleClasses(&cb);
                        cb.free();
                    }
                } break;
                case DEM_SignonPacket:
                case DEM_Packet: {
                    auto cb = handler.retrieve<msgDem::id>(msg.type, std::move(msg), msg.tick);
                    handlePacket(&cb);
                    cb.free();
                } break;
                case DEM_SendTables: {
                    if (set.parse_entities) {
                        auto cb = handler.retrieve<msgDem::id>(msg.type, std::move(msg), msg.tick);
                        handleSendTables(&cb);
                        cb.free();
                    }
                } break;
            }
            #else // _MSC_VER
                handler.forward<msgDem>(msg.type, std::move(msg), msg.tick);
            #endif // _MSC_VER
        }

        if (!stream->good())
            handler.forward<msgStatus>(REPLAY_FINISH, REPLAY_FINISH, tick);
    }

    void parser::handle() {
        // parse all messages while there is data available
        while (stream->good()) {
            read();
        }

        // let handlers know we are done
        handler.forward<msgStatus>(REPLAY_FINISH, REPLAY_FINISH, tick);
    }

    void parser::skipTo(uint32_t second) {
        uint32_t min = second/60;
        int32_t sec = second%60;

        // make sure we have a valid state before skipping ahead / back
        while (tick < 30) {
            read();
        }

        // clear all entities
        entities.clear();
        entities.resize(DOTA_MAX_ENTITIES);

        // skip to the fullpacket
        stream->move(min);

        // get the next full package in the stram
        demMessage_t msg;
        do {
            msg = stream->read(false);
        } while (msg.type != 13 && good());

        // parse it
        auto full = handler.retrieve<msgDem::id>(msg.type, std::move(msg), msg.tick);
        CDemoFullPacket* p = full.get<CDemoFullPacket>();

        for (auto &tbl : p->string_table().tables()) {
            auto it = stringtables.findKey(tbl.table_name());
            if (it == stringtables.end())
                continue;

            for (auto &item : tbl.items()) {
                it->value.set(item.str(), item.data());
            }

            for (auto &item : tbl.items_clientside()) {
                it->value.set(item.str(), item.data());
            }
        }

        // forward packets
        const std::string &data = p->packet().data();
        forwardMessageContainer<msgNet>(data.c_str(), data.size(), msg.tick);

        // asume 1 tick 2 / secs
        for (; sec > 0; sec -= 2) {
            read();
        }
    }

    handler_t* parser::getHandler() {
        return &handler;
    }

    const flatsendtable& parser::getFlattable(const std::string &tbl) {
        auto it = flattables.find(tbl);
        if (it == flattables.end())
            BOOST_THROW_EXCEPTION( sendtableUnkownTable()
                << EArg<1>::info(tbl)
            );

        return it->second;
    }

    uint32_t parser::getEntityIdFor(std::string name) {
        for (auto &e : clist) {<commit_msg>Fixed skipping using retrieve, compiles on windows now<commit_after>/**
 * @file parser.cpp
 * @author <NAME> <me (at) invokr (dot) org>
 * @version 1.0
 *
 * @par License
 *    Alice Replay Parser
 *    Copyright 2014 <NAME>
 *
 *    Licensed under the Apache License, Version 2.0 (the "License");
 *    you may not use this file except in compliance with the License.
 *    You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 *    Unless required by applicable law or agreed to in writing, software
 *    distributed under the License is distributed on an "AS IS" BASIS,
 *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *    See the License for the specific language governing permissions and
 *    limitations under the License.
 */

#include <alice/demo.pb.h>
#include <alice/netmessages.pb.h>
#include <alice/usermessages.pb.h>
#include <alice/dota_usermessages.pb.h>

#include <alice/bitstream.hpp>
#include <alice/parser.hpp>

namespace dota {
    parser::parser(const settings s, dem_stream *stream) : set(s), stream(stream), tick(0), msgs(0), sendtableId(-1),
        stringtableId(-1), delta(nullptr)
    {
        handlerRegisterCallback((&handler), msgDem, DEM_Packet,       parser, handlePacket)
        handlerRegisterCallback((&handler), msgDem, DEM_SignonPacket, parser, handlePacket)
        handlerRegisterCallback((&handler), msgNet, svc_UserMessage,  parser, handleUserMessage)

        if (set.parse_entities) {
            handlerRegisterCallback((&handler),  msgDem, DEM_ClassInfo,  parser, handleClasses)
            handlerRegisterCallback((&handler),  msgDem, DEM_SendTables, parser, handleSendTables)

            handlerRegisterCallback((&handler), msgNet, svc_ServerInfo, parser, handleServerInfo)
            handlerRegisterCallback((&handler), msgNet, svc_SendTable,  parser, handleSendTable)
        }

        if (set.parse_stringtables) {
            handlerRegisterCallback((&handler), msgNet, svc_CreateStringTable, parser, handleCreateStringtable)
            handlerRegisterCallback((&handler), msgNet, svc_UpdateStringTable, parser, handleUpdateStringtable)
        }

        if (set.parse_entities) {
            // allocate memory for entities
            entities.resize(DOTA_MAX_ENTITIES);

            // callback handlers
            handlerRegisterCallback((&handler), msgNet, svc_PacketEntities, parser, handleEntity)
        }

        if (set.track_entities) {
            delta = new entity_delta;
        }

        // Get unique / non-unique IDs for all messages
        registerTypes();
    }

    parser::~parser() {
        // free stream memory
        delete stream;

        // free sendprop table
        for (auto &tbl : sendtables) {
            tbl.value.free();
        }

        // free entity delta
        if (set.track_entities)
            delete delta;
    }

    void parser::open(std::string path) {
        file = path;
        stream->open(path);

        // let handlers know we begin to parse
        handler.forward<msgStatus>(REPLAY_START, REPLAY_START, 0);
    }

    bool parser::good() {
        return stream->good();
    }

    void parser::read() {
        // Read a single message
        //
        // If forward_dem is set, packages not handled by the parser are skipped because no
        // one would get them anyway.
        // This increases the read performance by about 20% for the DEM part.
        demMessage_t msg = stream->read(!set.forward_dem);
        ++msgs;

        // update current tick
        if (msg.tick > 0) // last ticks are 0ed
            tick = msg.tick;

        // Forward messages via the handler or handle them internaly
        if (set.forward_dem) {
            handler.forward<msgDem>(msg.type, std::move(msg), msg.tick);
        } else {
            #ifndef _MSC_VER
            switch (msg.type) {
                case DEM_ClassInfo: {
                    if (set.parse_entities) {
                        auto cb = handler.retrieve<msgDem::id>(msg.type, std::move(msg), msg.tick);
                        handleClasses(&cb);
                        cb.free();
                    }
                } break;
                case DEM_SignonPacket:
                case DEM_Packet: {
                    auto cb = handler.retrieve<msgDem::id>(msg.type, std::move(msg), msg.tick);
                    handlePacket(&cb);
                    cb.free();
                } break;
                case DEM_SendTables: {
                    if (set.parse_entities) {
                        auto cb = handler.retrieve<msgDem::id>(msg.type, std::move(msg), msg.tick);
                        handleSendTables(&cb);
                        cb.free();
                    }
                } break;
            }
            #else // _MSC_VER
                handler.forward<msgDem>(msg.type, std::move(msg), msg.tick);
            #endif // _MSC_VER
        }

        if (!stream->good())
            handler.forward<msgStatus>(REPLAY_FINISH, REPLAY_FINISH, tick);
    }

    void parser::handle() {
        // parse all messages while there is data available
        while (stream->good()) {
            read();
        }

        // let handlers know we are done
        handler.forward<msgStatus>(REPLAY_FINISH, REPLAY_FINISH, tick);
    }

    void parser::skipTo(uint32_t second) {
        uint32_t min = second/60;
        int32_t sec = second%60;

        // make sure we have a valid state before skipping ahead / back
        while (tick < 30) {
            read();
        }

        // clear all entities
        entities.clear();
        entities.resize(DOTA_MAX_ENTITIES);

        // skip to the fullpacket
        stream->move(min);

        // get the next full package in the stram
        demMessage_t msg;
        do {
            msg = stream->read(false);
        } while (msg.type != 13 && good());

        // parse it
        CDemoFullPacket p;
        p.ParseFromArray(msg.msg, msg.size);

        for (auto &tbl : p.string_table().tables()) {
            auto it = stringtables.findKey(tbl.table_name());
            if (it == stringtables.end())
                continue;

            for (auto &item : tbl.items()) {
                it->value.set(item.str(), item.data());
            }

            for (auto &item : tbl.items_clientside()) {
                it->value.set(item.str(), item.data());
            }
        }

        // forward packets
        const std::string &data = p.packet().data();
        forwardMessageContainer<msgNet>(data.c_str(), data.size(), msg.tick);

        // asume 1 tick 2 / secs
        for (; sec > 0; sec -= 2) {
            read();
        }
    }

    handler_t* parser::getHandler() {
        return &handler;
    }

    const flatsendtable& parser::getFlattable(const std::string &tbl) {
        auto it = flattables.find(tbl);
        if (it == flattables.end())
            BOOST_THROW_EXCEPTION( sendtableUnkownTable()
                << EArg<1>::info(tbl)
            );

        return it->second;
    }

    uint32_t parser::getEntityIdFor(std::string name) {
        for (auto &e : clist) {
{'fraction_non_alphanumeric': 0.1131786187190745, 'fraction_numerical': 0.005575301414732734, 'mean_word_length': 1.898404362754999, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 69, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7017846', 'n_tokens_mistral': 4117, 'n_tokens_neox': 3920, 'n_words': 1031}