<commit_before>/*
 * helix-svm - Order book events for libsvm
 *
 * This is a program that listens to a market data feed and outputs order book
 * events in libsvm compatible format.
 *
 * Feature extraction and classification model is based on the following paper:
 *
 *   Kercheval, <NAME>., and <NAME>. "Modeling high-frequency limit order
 *   book dynamics with support vector machines." (2013).
 */

#include <helix-c/helix.h>
#include <sys/mman.h>
#include <getopt.h>
#include <libgen.h>
#include <iostream>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <cerrno>
#include <cstdio>
#include <memory>
#include <vector>
#include <deque>
#include <cmath>
#include <tuple>
#include <uv.h>

using namespace std;

static const char *program;

struct config {
	const char *symbol;
	size_t max_orders;
	const char *proto;
	const char *multicast_addr;
	int multicast_port;
	const char *input;
	const char *output;
};

class svm_session {
	static constexpr size_t nr_levels = 10;

	struct features {
		std::vector<double> values;

		void add(double v) {
			values.push_back(v);
		}
	};

	// Type of a time point (in number of events):
	using time_point = uint64_t;

	// Type of duration (in number of events):
	using duration = uint64_t;

	// Type of mid price movement:
	enum class label_type : int {
		upward		= +1,
		downward	= -1,
		stationary	= 0,
	};

	// Look-ahead interval ahead of current time:
	static constexpr duration lookahead_interval = 5;

	// Lookback buffer of (midprice, features) tuple before current time:
	deque<tuple<uint64_t, features>> lookback;

	ostream& _output;
public:
	explicit svm_session(ostream& output)
		: _output{output}
	{ }
	void process_ob_event(helix_order_book_t ob) {
		if (helix_order_book_ask_levels(ob) < nr_levels && helix_order_book_bid_levels(ob) < nr_levels) {
			return;
		}
		uint64_t midprice = helix_order_book_midprice(ob, 0);
		auto&& curr_features = extract(ob);
		if (lookback.empty()) {
			lookback.emplace_back(make_tuple(midprice, curr_features));
			return;
		}
		uint64_t prev_midprice = helix_order_book_midprice(ob, 0);
		features prev_features;
		tie(prev_midprice, prev_features) = lookback.front();

		if (curr_features.values == prev_features.values) {
			return;
		}
		lookback.emplace_back(make_tuple(midprice, curr_features));

		if (lookback.size() < lookahead_interval) {
			return;
		}
		lookback.pop_front();

		fmt(label(prev_midprice, midprice), prev_features);
	}
private:
	features extract(helix_order_book_t ob) {
		features result{};
		double sum_bid_price = 0.0;
		double sum_ask_price = 0.0;
		double sum_bid_size = 0.0;
		double sum_ask_size = 0.0;
		for (size_t i = 0; i < nr_levels; i++) {
			auto bid_price = helix_order_book_bid_price(ob, i) / 10000.0;
			auto ask_price = helix_order_book_ask_price(ob, i) / 10000.0;
			auto bid_size = helix_order_book_bid_size(ob, i);
			auto ask_size = helix_order_book_ask_size(ob, i);
			auto midprice = helix_order_book_midprice(ob, i) / 10000.0;
			// Price and volume (n levels):
			result.add(ask_price);
			result.add(ask_size);
			result.add(bid_price);
			result.add(bid_size);
			// Bid-ask spread and mid-prices:
			result.add(ask_price - bid_price);
			result.add(midprice);
			// Mean prices and differences:
			sum_bid_price += bid_price;
			sum_ask_price += ask_price;
			sum_bid_size += bid_size;
			sum_ask_size += ask_size;
		}
		// Mean prices and differences:
		result.add(sum_bid_price / nr_levels);
		result.add(sum_ask_price / nr_levels);
		result.add(sum_bid_size / nr_levels);
		result.add(sum_ask_size / nr_levels);
		for (size_t i = 1; i < nr_levels; i++) {
			// Price differences:
			auto bid0 = helix_order_book_midprice(ob, i-1) / 10000.0;
			auto bid1 = helix_order_book_midprice(ob, i) / 10000.0;
			auto ask0 = helix_order_book_midprice(ob, i-1) / 10000.0;
			auto ask1 = helix_order_book_midprice(ob, i) / 10000.0;
			result.add(abs(ask1-ask0));
			result.add(abs(bid1-bid0));
		}
		return result;
	}

	label_type label(uint64_t prev_midprice, uint64_t midprice) {
		if (midprice > prev_midprice) {
			return label_type::upward;
		} else if (midprice < prev_midprice) {
			return label_type::downward;
		} else {
			return label_type::stationary;
		}
	}

	void fmt(label_type label, const features& features) {
		_output << int(label);
		for (size_t i = 0; i < features.values.size(); i++) {
			_output << " " << i+1 << ":" << features.values[i];
		}
		_output << endl;
	}
};

static void alloc_packet(uv_handle_t* handle, size_t size, uv_buf_t* buf)
{
	static char rx_buffer[65536];

	buf->base = rx_buffer;
	buf->len = sizeof(rx_buffer);
}

struct svm_fmt_ops *fmt_ops;

static void process_ob_event(helix_session_t session, helix_order_book_t ob)
{
	struct svm_session* s = reinterpret_cast<struct svm_session*>(helix_session_data(session));

	s->process_ob_event(ob);
}

static void process_event(helix_session_t session, helix_event_t event)
{
	helix_event_mask_t mask = helix_event_mask(event);

	if (mask & HELIX_EVENT_ORDER_BOOK_UPDATE) {
		helix_order_book_t ob = helix_event_order_book(event);
		process_ob_event(session, ob);
	}
}

static void recv_packet(uv_udp_t* handle, ssize_t nread, const uv_buf_t* buf, const struct sockaddr* addr, unsigned flags)
{
	if (nread > 0) {
		helix_session_process_packet(reinterpret_cast<helix_session_t>(handle->data), buf->base, nread);
	}
}

static void libuv_error(const char *s, int err)
{
	fprintf(stderr, "error: %s: %s (%s)\n", s, uv_strerror(err), uv_err_name(err));
	exit(1);<commit_msg>tools/helix-svm: Use consistent tag type<commit_after>/*
 * helix-svm - Order book events for libsvm
 *
 * This is a program that listens to a market data feed and outputs order book
 * events in libsvm compatible format.
 *
 * Feature extraction and classification model is based on the following paper:
 *
 *   Kercheval, <NAME>., and <NAME>. "Modeling high-frequency limit order
 *   book dynamics with support vector machines." (2013).
 */

#include <helix-c/helix.h>
#include <sys/mman.h>
#include <getopt.h>
#include <libgen.h>
#include <iostream>
#include <cstdlib>
#include <cstring>
#include <fstream>
#include <cerrno>
#include <cstdio>
#include <memory>
#include <vector>
#include <deque>
#include <cmath>
#include <tuple>
#include <uv.h>

using namespace std;

static const char *program;

struct config {
	const char *symbol;
	size_t max_orders;
	const char *proto;
	const char *multicast_addr;
	int multicast_port;
	const char *input;
	const char *output;
};

class svm_session {
	static constexpr size_t nr_levels = 10;

	struct features {
		std::vector<double> values;

		void add(double v) {
			values.push_back(v);
		}
	};

	// Type of a time point (in number of events):
	using time_point = uint64_t;

	// Type of duration (in number of events):
	using duration = uint64_t;

	// Type of mid price movement:
	enum class label_type : int {
		upward		= +1,
		downward	= -1,
		stationary	= 0,
	};

	// Look-ahead interval ahead of current time:
	static constexpr duration lookahead_interval = 5;

	// Lookback buffer of (midprice, features) tuple before current time:
	deque<tuple<uint64_t, features>> lookback;

	ostream& _output;
public:
	explicit svm_session(ostream& output)
		: _output{output}
	{ }
	void process_ob_event(helix_order_book_t ob) {
		if (helix_order_book_ask_levels(ob) < nr_levels && helix_order_book_bid_levels(ob) < nr_levels) {
			return;
		}
		uint64_t midprice = helix_order_book_midprice(ob, 0);
		auto&& curr_features = extract(ob);
		if (lookback.empty()) {
			lookback.emplace_back(make_tuple(midprice, curr_features));
			return;
		}
		uint64_t prev_midprice = helix_order_book_midprice(ob, 0);
		features prev_features;
		tie(prev_midprice, prev_features) = lookback.front();

		if (curr_features.values == prev_features.values) {
			return;
		}
		lookback.emplace_back(make_tuple(midprice, curr_features));

		if (lookback.size() < lookahead_interval) {
			return;
		}
		lookback.pop_front();

		fmt(label(prev_midprice, midprice), prev_features);
	}
private:
	features extract(helix_order_book_t ob) {
		features result{};
		double sum_bid_price = 0.0;
		double sum_ask_price = 0.0;
		double sum_bid_size = 0.0;
		double sum_ask_size = 0.0;
		for (size_t i = 0; i < nr_levels; i++) {
			auto bid_price = helix_order_book_bid_price(ob, i) / 10000.0;
			auto ask_price = helix_order_book_ask_price(ob, i) / 10000.0;
			auto bid_size = helix_order_book_bid_size(ob, i);
			auto ask_size = helix_order_book_ask_size(ob, i);
			auto midprice = helix_order_book_midprice(ob, i) / 10000.0;
			// Price and volume (n levels):
			result.add(ask_price);
			result.add(ask_size);
			result.add(bid_price);
			result.add(bid_size);
			// Bid-ask spread and mid-prices:
			result.add(ask_price - bid_price);
			result.add(midprice);
			// Mean prices and differences:
			sum_bid_price += bid_price;
			sum_ask_price += ask_price;
			sum_bid_size += bid_size;
			sum_ask_size += ask_size;
		}
		// Mean prices and differences:
		result.add(sum_bid_price / nr_levels);
		result.add(sum_ask_price / nr_levels);
		result.add(sum_bid_size / nr_levels);
		result.add(sum_ask_size / nr_levels);
		for (size_t i = 1; i < nr_levels; i++) {
			// Price differences:
			auto bid0 = helix_order_book_midprice(ob, i-1) / 10000.0;
			auto bid1 = helix_order_book_midprice(ob, i) / 10000.0;
			auto ask0 = helix_order_book_midprice(ob, i-1) / 10000.0;
			auto ask1 = helix_order_book_midprice(ob, i) / 10000.0;
			result.add(abs(ask1-ask0));
			result.add(abs(bid1-bid0));
		}
		return result;
	}

	label_type label(uint64_t prev_midprice, uint64_t midprice) {
		if (midprice > prev_midprice) {
			return label_type::upward;
		} else if (midprice < prev_midprice) {
			return label_type::downward;
		} else {
			return label_type::stationary;
		}
	}

	void fmt(label_type label, const features& features) {
		_output << int(label);
		for (size_t i = 0; i < features.values.size(); i++) {
			_output << " " << i+1 << ":" << features.values[i];
		}
		_output << endl;
	}
};

static void alloc_packet(uv_handle_t* handle, size_t size, uv_buf_t* buf)
{
	static char rx_buffer[65536];

	buf->base = rx_buffer;
	buf->len = sizeof(rx_buffer);
}

struct svm_fmt_ops *fmt_ops;

static void process_ob_event(helix_session_t session, helix_order_book_t ob)
{
	class svm_session* s = reinterpret_cast<class svm_session*>(helix_session_data(session));

	s->process_ob_event(ob);
}

static void process_event(helix_session_t session, helix_event_t event)
{
	helix_event_mask_t mask = helix_event_mask(event);

	if (mask & HELIX_EVENT_ORDER_BOOK_UPDATE) {
		helix_order_book_t ob = helix_event_order_book(event);
		process_ob_event(session, ob);
	}
}

static void recv_packet(uv_udp_t* handle, ssize_t nread, const uv_buf_t* buf, const struct sockaddr* addr, unsigned flags)
{
	if (nread > 0) {
		helix_session_process_packet(reinterpret_cast<helix_session_t>(handle->data), buf->base, nread);
	}
}

static void libuv_error(const char *s, int err)
{
	fprintf(stderr, "error: %s: %s (%s)\n", s, uv_strerror(err), uv_err_name(err));
	exit(1);
{'fraction_non_alphanumeric': 0.12099033161651758, 'fraction_numerical': 0.017529592482154153, 'mean_word_length': 4.433480608738341, 'pattern_counts': {'":': 2, '<': 87, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7460006', 'n_tokens_mistral': 4566, 'n_tokens_neox': 4091, 'n_words': 969}