<commit_before>
#include "bhget.h"

#include <iostream>
#include <list>
#include <sstream>
#include <utility>

#include "buildconf.hpp"

namespace asio = boost::asio;
namespace po = boost::program_options;
using namespace std;

using namespace bithorde;

const static size_t BLOCK_SIZE = (64*1024);

struct OutQueue {
	typedef pair<uint64_t, string> Chunk;
	uint64_t position;
	list<Chunk> _stored;

	OutQueue() :
		position(0)
	{}

	void send(uint64_t offset, const string& data) {
		if (offset <= position) {
			BOOST_ASSERT(offset == position);
			_flush(data);
			_dequeue();
		} else {
			_queue(offset, data);
		}
	}

private:
	void _queue(uint64_t offset, const string& data) {
		list<Chunk>::iterator pos = _stored.begin();
		while ((pos != _stored.end()) && (pos->first < offset))
			pos++;
		_stored.insert(pos, Chunk(offset, data));
	}

	void _dequeue() {
		while (_stored.size()) {
			Chunk& first = _stored.front();
			if (first.first > position) {
				break;
			} else {
				BOOST_ASSERT(first.first == position);
				_flush(first.second);
				_stored.pop_front();
			}
		}
	}

	void _flush(const string& data) {
		ssize_t datasize = data.size();
		if (write(1, data.data(), datasize) == datasize)
			position += datasize;
		else
			(cerr << "Error: failed to write block" << endl).flush();
	}
};

BHGet::BHGet(po::variables_map& args) :
	optMyName(args["name"].as<string>()),
	optQuiet(args.count("quiet")),
	optConnectUrl(args["url"].as<string>()),
	_ioSvc(),
	_asset(NULL)
{
}

int BHGet::main(const std::vector<std::string>& args) {
	std::vector<std::string>::const_iterator iter;
	for (iter = args.begin(); iter != args.end(); iter++) {
		if (!queueAsset(*iter))
			return 1;
	}

	_client = Client::create(_ioSvc, optMyName);
	_client->authenticated.connect(boost::bind(&BHGet::onAuthenticated, this, _1));
	_client->connect(optConnectUrl);

	_ioSvc.run();

	return 0;
}

bool BHGet::queueAsset(const std::string& _uri) {
	MagnetURI uri;
	if (!uri.parse(_uri)) {
		cerr << "Only magnet-links supported, not '" << _uri << "'" << endl;
		return false;
	}

	if (uri.xtIds.size()) {<commit_msg>[lib/bhget]Add symlink-resolution feature.<commit_after>
#include "bhget.h"

#include <iostream>
#include <list>
#include <sstream>
#include <utility>

#include "buildconf.hpp"

namespace asio = boost::asio;
namespace po = boost::program_options;
using namespace std;

using namespace bithorde;

const static size_t BLOCK_SIZE = (64*1024);

struct OutQueue {
	typedef pair<uint64_t, string> Chunk;
	uint64_t position;
	list<Chunk> _stored;

	OutQueue() :
		position(0)
	{}

	void send(uint64_t offset, const string& data) {
		if (offset <= position) {
			BOOST_ASSERT(offset == position);
			_flush(data);
			_dequeue();
		} else {
			_queue(offset, data);
		}
	}

private:
	void _queue(uint64_t offset, const string& data) {
		list<Chunk>::iterator pos = _stored.begin();
		while ((pos != _stored.end()) && (pos->first < offset))
			pos++;
		_stored.insert(pos, Chunk(offset, data));
	}

	void _dequeue() {
		while (_stored.size()) {
			Chunk& first = _stored.front();
			if (first.first > position) {
				break;
			} else {
				BOOST_ASSERT(first.first == position);
				_flush(first.second);
				_stored.pop_front();
			}
		}
	}

	void _flush(const string& data) {
		ssize_t datasize = data.size();
		if (write(1, data.data(), datasize) == datasize)
			position += datasize;
		else
			(cerr << "Error: failed to write block" << endl).flush();
	}
};

BHGet::BHGet(po::variables_map& args) :
	optMyName(args["name"].as<string>()),
	optQuiet(args.count("quiet")),
	optConnectUrl(args["url"].as<string>()),
	_ioSvc(),
	_asset(NULL)
{
}

int BHGet::main(const std::vector<std::string>& args) {
	std::vector<std::string>::const_iterator iter;
	for (iter = args.begin(); iter != args.end(); iter++) {
		if (!queueAsset(*iter))
			return 1;
	}

	_client = Client::create(_ioSvc, optMyName);
	_client->authenticated.connect(boost::bind(&BHGet::onAuthenticated, this, _1));
	_client->connect(optConnectUrl);

	_ioSvc.run();

	return 0;
}

bool resolvePath(MagnetURI& uri, const std::string &path_) {
	char pathbuf[PATH_MAX];
	auto path__ = realpath(path_.c_str(), pathbuf);
	if (!path__) {
		return false;
	} else {
		boost::filesystem::path p(path__);
		return uri.parse(p.filename().string());
	}
}

bool BHGet::queueAsset(const std::string& _uri) {
	MagnetURI uri;
	if (!uri.parse(_uri) && !resolvePath(uri, _uri)) {
		cerr << "Only magnet-links and symlinks to magnet-links supported, not '" << _uri << "'" << endl;
		return false;
	}

	if (uri.xtIds.size()) {
{'fraction_non_alphanumeric': 0.1754270696452037, 'fraction_numerical': 0.008322382829610162, 'mean_word_length': 4.54247572815534, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6154046', 'n_tokens_mistral': 1852, 'n_tokens_neox': 1677, 'n_words': 363}