<commit_before>#ifndef _INCLUDED_CHAIN_COMPRESSOR_HPP_
#define _INCLUDED_CHAIN_COMPRESSOR_HPP_

#include <tudocomp/Compressor.hpp>
#include <tudocomp/Env.hpp>
#include <tudocomp/io.hpp>
#include <tudocomp/CreateAlgorithm.hpp>
#include <vector>
#include <memory>

namespace tdc {

class ChainCompressor: public Compressor {
public:
    inline static Meta meta() {
        Meta m("compressor", "chain");
        m.option("first").dynamic_compressor();
        m.option("second").dynamic_compressor();
        return m;
    }

    /// No default construction allowed
    inline ChainCompressor() = delete;

    /// Construct the class with an environment and the algorithms to chain.
    inline ChainCompressor(Env&& env):
        Compressor(std::move(env)) {}

    template<class F>
    inline void chain(Input& input, Output& output, bool reverse, F f) {
        string_ref first_algo = "first";
        string_ref second_algo = "second";
        if (reverse) {
            std::swap(first_algo, second_algo);
        }

        auto run = [&](Input& i, Output& o, string_ref option) {
            auto& option_value = env().option(option);
            DCHECK(option_value.is_algorithm());

            auto av = option_value.as_algorithm();

            bool needs_sentinel = av.needs_sentinel_terminator();

            // TODO: Debug!
            //needs_sentinel = true;

            auto compressor = create_algo_with_registry_dynamic(
                env().registry(), av);

            f(i, o, *compressor, needs_sentinel);
        };

        std::vector<uint8_t> between_buf;
        {
            Output between(between_buf);
            run(input, between, first_algo);
        }
        // TODO: Debug!
        //std::cout << vec_to_debug_string(between_buf) << "\n";
        {
            Input between(between_buf);
            run(between, output, second_algo);
        }
    }

    /// Compress `inp` into `out`.
    ///
    /// \param input The input stream.
    /// \param output The output stream.
    inline virtual void compress(Input& input, Output& output) override final {
        chain(input, output, false, [](Input& i, Output& o, Compressor& c, bool needs_sentinel) {
            if (needs_sentinel) {
                i.escape_and_terminate();
            }
            c.compress(i, o);
        });
    }

    /// Decompress `inp` into `out`.
    ///
    /// \param input The input stream.
    /// \param output The output stream.
    inline virtual void decompress(Input& input, Output& output) override final {
        chain(input, output, true, [](Input& i, Output& o, Compressor& c, bool needs_sentinel) {
            if (needs_sentinel) {
                o.unescape_and_trim();
            }
            c.decompress(i, o);
        });
    }
};

}<commit_msg>more debug<commit_after>#ifndef _INCLUDED_CHAIN_COMPRESSOR_HPP_
#define _INCLUDED_CHAIN_COMPRESSOR_HPP_

#include <tudocomp/Compressor.hpp>
#include <tudocomp/Env.hpp>
#include <tudocomp/io.hpp>
#include <tudocomp/CreateAlgorithm.hpp>
#include <vector>
#include <memory>

namespace tdc {

class ChainCompressor: public Compressor {
public:
    inline static Meta meta() {
        Meta m("compressor", "chain");
        m.option("first").dynamic_compressor();
        m.option("second").dynamic_compressor();
        return m;
    }

    /// No default construction allowed
    inline ChainCompressor() = delete;

    /// Construct the class with an environment and the algorithms to chain.
    inline ChainCompressor(Env&& env):
        Compressor(std::move(env)) {}

    template<class F>
    inline void chain(Input& input, Output& output, bool reverse, F f) {
        string_ref first_algo = "first";
        string_ref second_algo = "second";
        if (reverse) {
            std::swap(first_algo, second_algo);
        }

        auto run = [&](Input& i, Output& o, string_ref option) {
            auto& option_value = env().option(option);
            DCHECK(option_value.is_algorithm());

            auto av = option_value.as_algorithm();

            bool needs_sentinel = av.needs_sentinel_terminator();

            std::cout << "\n";
            std::cout << "Algo: " << av.name() << "\n";
            std::cout << "Needs sentinel: " << (needs_sentinel ? "true" : "false") << "\n";

            // TODO: Debug!
            //needs_sentinel = true;

            auto compressor = create_algo_with_registry_dynamic(
                env().registry(), av);

            f(i, o, *compressor, needs_sentinel);
        };

        std::vector<uint8_t> between_buf;
        std::cout << "Before chain\n";
        {
            Output between(between_buf);
            run(input, between, first_algo);
        }
        // TODO: Debug!
        std::cout << "Between chains: " << vec_to_debug_string(between_buf) << "\n";
        {
            Input between(between_buf);
            run(between, output, second_algo);
        }
        std::cout << "After chain\n";
    }

    /// Compress `inp` into `out`.
    ///
    /// \param input The input stream.
    /// \param output The output stream.
    inline virtual void compress(Input& input, Output& output) override final {
        chain(input, output, false, [](Input& i, Output& o, Compressor& c, bool needs_sentinel) {
            if (needs_sentinel) {
                i.escape_and_terminate();
            }
            c.compress(i, o);
        });
    }

    /// Decompress `inp` into `out`.
    ///
    /// \param input The input stream.
    /// \param output The output stream.
    inline virtual void decompress(Input& input, Output& output) override final {
        chain(input, output, true, [](Input& i, Output& o, Compressor& c, bool needs_sentinel) {
            if (needs_sentinel) {
                o.unescape_and_trim();
            }
            c.decompress(i, o);
        });
    }
};

}
{'fraction_non_alphanumeric': 0.1294398907103825, 'fraction_numerical': 0.00034153005464480874, 'mean_word_length': 2.164235548352242, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1349224', 'n_tokens_mistral': 1810, 'n_tokens_neox': 1731, 'n_words': 430}