<commit_before>#ifndef ALGORITHM_HUFFMAN_H_
#define ALGORITHM_HUFFMAN_H_ 1

#include <vector>
#include <array>
#include <limits>

namespace algorithm
{

/** \brief  Modified Huffman coding

    Huffman coding method with Run Length Encoding.
    Combines the variable length codes of Huffman coding with the
    coding of repetitive data in Run Length Encoding.
*/
class Huffman
{
public:
    typedef size_t          SizeType;

    typedef std::ifstream   StreamInType;
    typedef std::ofstream   StreamOutType;

    typedef std::string     StringType;

    typedef uint8_t         ByteType;       /** One byte */
    typedef uint32_t        CodewordType;   /** Codeword, consists of 4 bytes */

    typedef std::pair<ByteType, SizeType> MetaSymbolType;

    /** Constants */
    static  const ByteType  zerobit     = 0;
    static  const ByteType  nonzerobit  = 1;
    static  const SizeType  ascii_max   = std::numeric_limits<ByteType>::max();

    static  const SizeType  byte_size   = 8;                /**< 8 bits, bit size of ByteType */
    static  const SizeType  buffer_size = byte_size * 4;    /**< Bit size of CodewordType */

    /** Class for each node in Huffman tree, and used in RLE */
    struct Run
    {
        /** MetaSymbol, paired with MetaSymbolType */
        ByteType        symbol;         /**< ASCII character */
        SizeType        run_len;        /**< Run-length, used in RLE */<commit_msg>Change the file stream to abstract stream<commit_after>#ifndef ALGORITHM_HUFFMAN_H_
#define ALGORITHM_HUFFMAN_H_ 1

#include <vector>
#include <array>
#include <limits>

namespace algorithm
{

/** \brief  Modified Huffman coding

    Huffman coding method with Run Length Encoding.
    Combines the variable length codes of Huffman coding with the
    coding of repetitive data in Run Length Encoding.
*/
class Huffman
{
public:
    typedef size_t          SizeType;

    typedef std::istream    StreamInType;
    typedef std::ostream    StreamOutType;

    typedef std::string     StringType;

    typedef uint8_t         ByteType;       /** One byte */
    typedef uint32_t        CodewordType;   /** Codeword, consists of 4 bytes */

    typedef std::pair<ByteType, SizeType> MetaSymbolType;

    /** Constants */
    static  const ByteType  zerobit     = 0;
    static  const ByteType  nonzerobit  = 1;
    static  const SizeType  ascii_max   = std::numeric_limits<ByteType>::max();

    static  const SizeType  byte_size   = 8;                /**< 8 bits, bit size of ByteType */
    static  const SizeType  buffer_size = byte_size * 4;    /**< Bit size of CodewordType */

    /** Class for each node in Huffman tree, and used in RLE */
    struct Run
    {
        /** MetaSymbol, paired with MetaSymbolType */
        ByteType        symbol;         /**< ASCII character */
        SizeType        run_len;        /**< Run-length, used in RLE */
{'fraction_non_alphanumeric': 0.08339124391938846, 'fraction_numerical': 0.006949270326615705, 'mean_word_length': 2.4896969696969697, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '811409', 'n_tokens_mistral': 886, 'n_tokens_neox': 823, 'n_words': 282}