<commit_before># pragma once

# include <typeinfo>
# include <vector>
# include <unordered_map>
# include <cstdint>

# include "goo_dataflow/processor.hpp"
# include "goo_dataflow/tier.hpp"
# include "goo_dataflow/framework.hpp"

namespace goo {
namespace dataflow {

/**@brief Thread local storage for a worker.
 * @class TLS
 *
 * ...
 * */
class Storage : public std::vector<uint8_t> {
private:
    std::vector<ValuesMap> * _vms;
protected:
    Storage( const Framework::Cache & );
    ~Storage();
    /// Builds variables map for certain processor in certain tier.
    ValuesMap & values_map_for( size_t tierNo, size_t processorNo );

    friend class Worker;
};

/** A worker constitues thread-local context to be utilized during graph
 * traversal.
 * */
class Worker {
public:
    struct LogEntry {
        static constexpr uint8_t execStarted = 0x1;
        static constexpr uint8_t execEnded = 0x2;

        uint8_t type;
        size_t nProc, nTier;
        std::clock_t time;
    };
protected:
    Framework & _fwRef;
    mutable std::mutex _logSyncM;
    std::vector <LogEntry> _log;
public:
    Worker( Framework & fr ) : _fwRef(fr) {}
    /// Must be passed to a std::thread.
    void run();
    /// Log. TODO: sync access.
    const std::vector<LogEntry> log() const {
        std::unique_lock<std::mutex> l(_logSyncM);
        return _log;
    }
};

}  // namespace goo::dataflow
}  // namespace goo

<commit_msg>+ docs<commit_after># pragma once

# include <typeinfo>
# include <vector>
# include <unordered_map>
# include <cstdint>

# include "goo_dataflow/processor.hpp"
# include "goo_dataflow/tier.hpp"
# include "goo_dataflow/framework.hpp"

namespace goo {
namespace dataflow {

/**@brief Thread local storage for a worker.
 * @class Storage
 *
 * Implements a dynamic buffer providing thread-local storage for the data used
 * by links in DAG.
 *
 * @TODO custom allocation/deletion of data, lifecycle hooks, std::allocator
 * support
 * */
class Storage : public std::vector<uint8_t> {
private:
    std::vector<ValuesMap> * _vms;
protected:
    Storage( const Framework::Cache & );
    ~Storage();
    /// Builds variables map for certain processor in certain tier.
    ValuesMap & values_map_for( size_t tierNo, size_t processorNo );

    friend class Worker;
};

/**@class Worker
 * @brief Defines thread-local context to be utilized during graph traversal.
 * */
class Worker {
public:
    struct LogEntry {
        static constexpr uint8_t execStarted = 0x1;
        static constexpr uint8_t execEnded = 0x2;

        uint8_t type;
        size_t nProc, nTier;
        std::clock_t time;
    };
protected:
    Framework & _fwRef;
    mutable std::mutex _logSyncM;
    std::vector <LogEntry> _log;
public:
    Worker( Framework & fr ) : _fwRef(fr) {}
    /// Must be passed to a std::thread.
    void run();
    /// Log. TODO: sync access.
    const std::vector<LogEntry> log() const {
        std::unique_lock<std::mutex> l(_logSyncM);
        return _log;
    }
};

}  // namespace goo::dataflow
}  // namespace goo


{'fraction_non_alphanumeric': 0.11617984903183459, 'fraction_numerical': 0.005251066622907778, 'mean_word_length': 3.135685210312076, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5257281', 'n_tokens_mistral': 1008, 'n_tokens_neox': 979, 'n_words': 288}