<commit_before>#include "chlog.hpp"
#include "ch_frb_io_internals.hpp"

using namespace std;

namespace ch_frb_io {
#if 0
};  // pacify emacs c-mode!
#endif


// Static factory function 
shared_ptr<output_device> output_device::make(const output_device::initializer &ini_params)
{
    output_device *p = new output_device(ini_params);
    shared_ptr<output_device> ret(p);

    ret->output_thread = std::thread(std::bind(&output_device::io_thread_main, ret));
    return ret;
}


output_device::output_device(const output_device::initializer &ini_params_) :
    ini_params(ini_params_)
{
    // Note: no sanity-checking of ini_params needed here!
    // XXX 32M is overkill!  How much space should I use here?
    this->_buffer = unique_ptr<uint8_t[]> (new uint8_t[32 * 1024 * 1024]);
}


void output_device::io_thread_main()
{
    // Note: if 'io_thread_allowed_cores' is an empty list, no core-pinning will be done.
    pin_thread_to_cores(ini_params.io_thread_allowed_cores);

    // If io_thread_name is unspecified, assign a default thread name.
    string thread_name = ini_params.io_thread_name;
    if (thread_name.size() == 0)
	thread_name = "io_thread[" + ini_params.device_name + "]";

    chime_log_set_thread_name(thread_name);

    if (ini_params.verbosity >= 2)
	chlog("thread starting up");

    for (;;) {
	// Pull a write_chunk_request off the queue!
	shared_ptr<write_chunk_request> w = this->pop_write_request();
	if (!w)
	    break;

	shared_ptr<assembled_chunk> chunk = w->chunk;
	string error_message;
	string link_src;

	unique_lock<mutex> ulock(chunk->filename_mutex);

	// If this write request is a duplicate of a previous write request, return immediately.
	if (chunk->filename_set.count(w->filename) > 0) {
	    ulock.unlock();

	    if (ini_params.verbosity >= 3)
		chlog("write request '" + w->filename + "' is a duplicate, skipping...");

	    w->write_callback("");
	    return;
	}
	
	// If this write request is a "pseudo-duplicate", i.e. the same chunk has been
	// previously written to a different filename on the same output_device, then
	// we'll make a hard link instead of a copy.

	auto p = chunk->filename_map.find(ini_params.device_name);	
	if (p != chunk->filename_map.end())
	    link_src = p->second;

	ulock.unlock();<commit_msg>oops, 'return' -> 'continue' in output_device::io_thread_main()<commit_after>#include "chlog.hpp"
#include "ch_frb_io_internals.hpp"

using namespace std;

namespace ch_frb_io {
#if 0
};  // pacify emacs c-mode!
#endif


// Static factory function 
shared_ptr<output_device> output_device::make(const output_device::initializer &ini_params)
{
    output_device *p = new output_device(ini_params);
    shared_ptr<output_device> ret(p);

    ret->output_thread = std::thread(std::bind(&output_device::io_thread_main, ret));
    return ret;
}


output_device::output_device(const output_device::initializer &ini_params_) :
    ini_params(ini_params_)
{
    // Note: no sanity-checking of ini_params needed here!
    // XXX 32M is overkill!  How much space should I use here?
    this->_buffer = unique_ptr<uint8_t[]> (new uint8_t[32 * 1024 * 1024]);
}


void output_device::io_thread_main()
{
    // Note: if 'io_thread_allowed_cores' is an empty list, no core-pinning will be done.
    pin_thread_to_cores(ini_params.io_thread_allowed_cores);

    // If io_thread_name is unspecified, assign a default thread name.
    string thread_name = ini_params.io_thread_name;
    if (thread_name.size() == 0)
	thread_name = "io_thread[" + ini_params.device_name + "]";

    chime_log_set_thread_name(thread_name);

    if (ini_params.verbosity >= 2)
	chlog("thread starting up");

    for (;;) {
	// Pull a write_chunk_request off the queue!
	shared_ptr<write_chunk_request> w = this->pop_write_request();
	if (!w)
	    break;

	shared_ptr<assembled_chunk> chunk = w->chunk;
	string error_message;
	string link_src;

	unique_lock<mutex> ulock(chunk->filename_mutex);

	// If this write request is a duplicate of a previous write request, return immediately.
	if (chunk->filename_set.count(w->filename) > 0) {
	    ulock.unlock();

	    if (ini_params.verbosity >= 3)
		chlog("write request '" + w->filename + "' is a duplicate, skipping...");

	    w->write_callback("");
	    continue;
	}
	
	// If this write request is a "pseudo-duplicate", i.e. the same chunk has been
	// previously written to a different filename on the same output_device, then
	// we'll make a hard link instead of a copy.

	auto p = chunk->filename_map.find(ini_params.device_name);	
	if (p != chunk->filename_map.end())
	    link_src = p->second;

	ulock.unlock();
{'fraction_non_alphanumeric': 0.12223187147199305, 'fraction_numerical': 0.008250108554059922, 'mean_word_length': 4.182227221597301, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2310223', 'n_tokens_mistral': 1631, 'n_tokens_neox': 1522, 'n_words': 393}