<commit_before>#ifndef _INCLUDED_LZW_BIT_CODER_HPP_
#define _INCLUDED_LZW_BIT_CODER_HPP_

#include <tudocomp/Coder.hpp>

#include <tudocomp/lz78/trie.h>
#include <tudocomp/lz78/Lz78DecodeBuffer.hpp>

#include <tudocomp/lzw/factor.h>
#include <tudocomp/lzw/decode.hpp>

namespace tudocomp {

namespace lzw {

using ::lzw::LzwEntry;
using lz78_dictionary::CodeType;

/**
 * Encodes factors as simple strings.
 */
class LzwBitCoder {
private:
    // TODO: Change encode_* methods to not take Output& since that inital setup
    // rather, have a single init location
    tudocomp::io::OutputStreamGuard m_out_guard;
    tudocomp::BitOStream m_out;
    uint64_t m_factor_counter = 0;

public:
    inline LzwBitCoder(Env& env, Output& out)
        : m_out_guard(out.as_stream()), m_out(*m_out_guard)
    {
    }

    inline ~LzwBitCoder() {
        m_out.flush();
        (*m_out_guard).flush();
    }

    inline void encode_fact(const LzwEntry& entry) {
        // output format: variable_number_backref_bits 8bit_char

        // slowly grow the number of bits needed together with the output
        size_t back_ref_idx_bits = bitsFor(m_factor_counter + 256);

        DCHECK(bitsFor(entry) <= back_ref_idx_bits);

        m_out.write(entry, back_ref_idx_bits);

        m_factor_counter++;
    }

    inline void dictionary_reset() {
        m_factor_counter = 0;
    }

    inline static void decode(Input& _inp, Output& _out,
                              CodeType dms,
                              CodeType reserve_dms) {
        auto iguard = _inp.as_stream();
        auto oguard = _out.as_stream();
        auto& inp = *iguard;
        auto& out = *oguard;

        bool done = false;
        BitIStream is(inp, done);

        uint64_t counter = 0;
        decode_step([&](CodeType& entry, bool reset, bool &file_corrupted) -> LzwEntry {
            // Try to read next factor
            LzwEntry factor(is.readBits<uint64_t>(bitsFor(counter + 256)));
            if (done) {
                // Could not read all bits -> done
                // (this works because the encoded factors are always > 8 bit)
                return false;
            }
            counter++;<commit_msg>Fixed new template based bit decoder to reset dictionary<commit_after>#ifndef _INCLUDED_LZW_BIT_CODER_HPP_
#define _INCLUDED_LZW_BIT_CODER_HPP_

#include <tudocomp/Coder.hpp>

#include <tudocomp/lz78/trie.h>
#include <tudocomp/lz78/Lz78DecodeBuffer.hpp>

#include <tudocomp/lzw/factor.h>
#include <tudocomp/lzw/decode.hpp>

namespace tudocomp {

namespace lzw {

using ::lzw::LzwEntry;
using lz78_dictionary::CodeType;

/**
 * Encodes factors as simple strings.
 */
class LzwBitCoder {
private:
    // TODO: Change encode_* methods to not take Output& since that inital setup
    // rather, have a single init location
    tudocomp::io::OutputStreamGuard m_out_guard;
    tudocomp::BitOStream m_out;
    uint64_t m_factor_counter = 0;

public:
    inline LzwBitCoder(Env& env, Output& out)
        : m_out_guard(out.as_stream()), m_out(*m_out_guard)
    {
    }

    inline ~LzwBitCoder() {
        m_out.flush();
        (*m_out_guard).flush();
    }

    inline void encode_fact(const LzwEntry& entry) {
        // output format: variable_number_backref_bits 8bit_char

        // slowly grow the number of bits needed together with the output
        size_t back_ref_idx_bits = bitsFor(m_factor_counter + 256);

        DCHECK(bitsFor(entry) <= back_ref_idx_bits);

        m_out.write(entry, back_ref_idx_bits);

        m_factor_counter++;
    }

    inline void dictionary_reset() {
        m_factor_counter = 0;
    }

    inline static void decode(Input& _inp, Output& _out,
                              CodeType dms,
                              CodeType reserve_dms) {
        auto iguard = _inp.as_stream();
        auto oguard = _out.as_stream();
        auto& inp = *iguard;
        auto& out = *oguard;

        bool done = false;
        BitIStream is(inp, done);

        uint64_t counter = 0;
        decode_step([&](CodeType& entry, bool reset, bool &file_corrupted) -> LzwEntry {
            if (reset) {
                counter = 0;
            }

            // Try to read next factor
            LzwEntry factor(is.readBits<uint64_t>(bitsFor(counter + 256)));
            if (done) {
                // Could not read all bits -> done
                // (this works because the encoded factors are always > 8 bit)
                return false;
            }
            counter++;
{'fraction_non_alphanumeric': 0.09826589595375723, 'fraction_numerical': 0.01133837261004891, 'mean_word_length': 2.3599701269604183, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '804323', 'n_tokens_mistral': 1527, 'n_tokens_neox': 1462, 'n_words': 349}