<commit_before>#include <cstdio>
#include <cstdlib>
#include <cinttypes>
#include <iostream>
#include <iomanip>
#include <sstream>
#include <fstream>
#include <algorithm>
#include <functional>
#include <limits>
#include <memory>
#include <string>
#include <vector>
#include <deque>
#include <map>
#include <set>

#include <unistd.h>

#include "riscv-util.h"
#include "riscv-model.h"
#include "riscv-cmdline.h"
#include "riscv-color.h"

static bool enable_color = false;

#define _COLOR_OPCODE _COLOR_BEGIN _COLOR_UNDERSCORE _COLOR_SEP _COLOR_FG_YELLOW _COLOR_END
#define _COLOR_BITS   _COLOR_BEGIN _COLOR_REVERSE    _COLOR_SEP _COLOR_FG_GREEN  _COLOR_END
#define _COLOR_FORMAT _COLOR_BEGIN _COLOR_BOLD       _COLOR_SEP _COLOR_FG_RED    _COLOR_END
#define _COLOR_LEGEND _COLOR_BEGIN _COLOR_BOLD       _COLOR_END
#define _COLOR_EXT    _COLOR_BEGIN _COLOR_FG_RED     _COLOR_END

static const ssize_t kMaxInstructionWidth = 32;
static const ssize_t kLatexTableColumns = 32;
static const ssize_t kLatexTableRows = 52;
static const ssize_t kLatexReserveRows = 5;

static const char* kLatexDocumentBegin =
R"LaTeX(\documentclass{report}
\usepackage[letterpaper, portrait, margin=0.5in]{geometry}
\usepackage[utf8]{inputenc}
\begin{document}
)LaTeX";

static const char* kLatexDocumentEnd =
R"LaTeX(\end{document}
)LaTeX";

static const char* kLatexTableBegin =
R"LaTeX(\newpage
\begin{table}[p]
\begin{small}
\begin{center}
\begin{tabular})LaTeX";

static const char* kLatexTableEnd =
R"LaTeX(\end{tabular}
\end{center}
\end{small}
\end{table}
)LaTeX";

enum riscv_latex_type {
	riscv_latex_type_empty,
	riscv_latex_type_line,
	riscv_latex_type_page_break,
	riscv_latex_type_extension_heading,
	riscv_latex_type_extension_contd,
	riscv_latex_type_opcode,
	riscv_latex_type_type_bitrange,
	riscv_latex_type_type_spec,
};

struct riscv_latex_row
{
	riscv_latex_type row_type;
	riscv_extension_ptr extension;
	riscv_opcode_ptr opcode;
	riscv_type_ptr type;

	riscv_latex_row(riscv_latex_type row_type)
		: row_type(row_type) {}

	riscv_latex_row(riscv_latex_type row_type, riscv_extension_ptr extension)
		: row_type(row_type), extension(extension) {}

	riscv_latex_row(riscv_latex_type row_type, riscv_opcode_ptr opcode)
		: row_type(row_type), opcode(opcode) {}

	riscv_latex_row(riscv_latex_type row_type, riscv_type_ptr type)
		: row_type(row_type), type(type) {}
};

struct riscv_latex_page
{
	std::vector<riscv_latex_row> rows;
};

struct riscv_codec_node
{
	std::vector<ssize_t> bits;
	std::vector<ssize_t> vals;
	std::map<ssize_t,riscv_opcode_list> val_opcodes;
	std::map<ssize_t,riscv_codec_node> val_decodes;

	void clear();
};

struct riscv_parse_meta : riscv_meta_model
{
	riscv_extension_list     ext_subset;
	riscv_codec_node         root_node;
	riscv_opcode_ptr         unknown;

	void generate_map();
	void generate_codec(bool include_pseudo);

	std::string colorize_args(riscv_opcode_ptr opcode);
	std::vector<std::string> get_unique_codecs();

	void print_latex_row(riscv_latex_row &row, std::string ts);
	void print_latex();
	void print_map(bool print_map_instructions);
	void print_c_header(std::string filename);
	void print_opcodes_h(bool no_comment = false, bool zero_not_oh = false);
	void print_opcodes_c(bool no_comment = false, bool zero_not_oh = false);
	void print_args_h();
	void print_codec_h(bool no_comment = false, bool zero_not_oh = false);

	void generate_codec_node(riscv_codec_node &node, riscv_opcode_list &opcode_list);
	void print_switch_decoder_node(riscv_codec_node &node, size_t indent);
};

template <typename T>
std::string join(std::vector<T> list, std::string sep)
{
	std::stringstream ss;
	for (auto i = list.begin(); i != list.end(); i++) {
		ss << (i != list.begin() ? sep : "") << *i;
	}
	return ss.str();
}

void riscv_codec_node::clear()
{
	bits.clear();
	vals.clear();
	val_opcodes.clear();
	val_decodes.clear();
}

void riscv_parse_meta::generate_map()
{
	for (auto &opcode : opcodes) {
		for (auto &mask : opcode->masks) {
			ssize_t msb = mask.first.msb;
			ssize_t lsb = mask.first.lsb;
			ssize_t val = mask.second;
			for (ssize_t bit = msb; bit >= lsb; bit--) {
				opcode->mask |= (1 << bit);
				opcode->match |= (val >> (bit - lsb)) & 1 ? (1 << bit) : 0;
			}
		}
	}
}

void riscv_parse_meta::generate_codec(bool include_pseudo)
{
	// make list of opcodes to include
	riscv_opcode_list opcodes_copy;
	for (auto &opcode : opcodes) {
		if (include_pseudo || opcode->key.find("@") == std::string::npos) {
			opcodes_copy.push_back(opcode);
		}
	}

	// generate decode
	root_node.clear();
	generate_codec_node(root_node, opcodes_copy);
}

std::string riscv_parse_meta::colorize_args(riscv_opcode_ptr opcode)
{
	std::vector<char> token;
	std::vector<std::string> comps;
	auto args = opcode->format->args;

	for (size_t i = 0; i < args.length(); i++) {
		char c = args[i];
		if (::isalnum(c)) {
			token.push_back(c);
		} else {
			comps.push_back(std::string(token.begin(), token.end()));
			token.resize(0);
			token.push_back(c);
			comps.push_back(std::string(token.begin(), token.end()));
			token.resize(0);
		}
	}
	if (token.size() > 0) {
		comps.push_back(std::string(token.begin(), token.end()));
	}

	for (size_t i = 0; i < comps.size(); i++) {
		auto comp = comps[i];
		auto arg = args_by_name[comp];
		if (arg) {
			auto new_comp = riscv_colors_to_ansi_escape_sequence(
				arg->fg_color, arg->bg_color, ansi_color_normal
			);
			new_comp.append(comp);
			new_comp.append(_COLOR_RESET);
			comps[i] = new_comp;
		}
	}

	return join(comps, "");
}

std::vector<std::string> riscv_parse_meta::get_unique_codecs()
{
	std::vector<std::string> codec_names;
	for (auto &codec : codecs) {
		std::string codec_name = format_codec("", codec, "_");
		if (std::find(codec_names.begin(), codec_names.end(), codec_name) == codec_names.end()) {
			codec_names.push_back(codec_name);
		}
	}
	return codec_names;
}

static std::string latex_utf_substitute(std::string str)
{
	str = replace(str, "{", "\\left\\{");
	str = replace(str, "}", "\\right\\}");
	str = replace(str, "≠", "\\neq");
	return str;
}

void riscv_parse_meta::print_latex_row(riscv_latex_row &row, std::string ts)
{
	switch (row.row_type) {
		case riscv_latex_type_empty:
			printf("\\tabularnewline\n");
			break;
		case riscv_latex_type_line:
			printf("\\cline{1-%ld}\n", kLatexTableColumns);
			break;
		case riscv_latex_type_page_break:
			printf("%s", kLatexTableEnd);
			printf("%s%s", kLatexTableBegin, ts.c_str());
			break;
		case riscv_latex_type_extension_heading:
		case riscv_latex_type_extension_contd:
		{
			printf("\\multicolumn{%ld}{c}{\\bf %s %s} & \\\\\n\\cline{1-%ld}\n",
				kLatexTableColumns,
				row.extension->description.c_str(),
				row.row_type == riscv_latex_type_extension_contd ? " contd" : "",
				kLatexTableColumns);
			break;
		}
		case riscv_latex_type_opcode:<commit_msg>Add additional UTF-8 to LaTeX substitutions<commit_after>#include <cstdio>
#include <cstdlib>
#include <cinttypes>
#include <iostream>
#include <iomanip>
#include <sstream>
#include <fstream>
#include <algorithm>
#include <functional>
#include <limits>
#include <memory>
#include <string>
#include <vector>
#include <deque>
#include <map>
#include <set>

#include <unistd.h>

#include "riscv-util.h"
#include "riscv-model.h"
#include "riscv-cmdline.h"
#include "riscv-color.h"

static bool enable_color = false;

#define _COLOR_OPCODE _COLOR_BEGIN _COLOR_UNDERSCORE _COLOR_SEP _COLOR_FG_YELLOW _COLOR_END
#define _COLOR_BITS   _COLOR_BEGIN _COLOR_REVERSE    _COLOR_SEP _COLOR_FG_GREEN  _COLOR_END
#define _COLOR_FORMAT _COLOR_BEGIN _COLOR_BOLD       _COLOR_SEP _COLOR_FG_RED    _COLOR_END
#define _COLOR_LEGEND _COLOR_BEGIN _COLOR_BOLD       _COLOR_END
#define _COLOR_EXT    _COLOR_BEGIN _COLOR_FG_RED     _COLOR_END

static const ssize_t kMaxInstructionWidth = 32;
static const ssize_t kLatexTableColumns = 32;
static const ssize_t kLatexTableRows = 52;
static const ssize_t kLatexReserveRows = 5;

static const char* kLatexDocumentBegin =
R"LaTeX(\documentclass{report}
\usepackage[letterpaper, portrait, margin=0.5in]{geometry}
\usepackage[utf8]{inputenc}
\begin{document}
)LaTeX";

static const char* kLatexDocumentEnd =
R"LaTeX(\end{document}
)LaTeX";

static const char* kLatexTableBegin =
R"LaTeX(\newpage
\begin{table}[p]
\begin{small}
\begin{center}
\begin{tabular})LaTeX";

static const char* kLatexTableEnd =
R"LaTeX(\end{tabular}
\end{center}
\end{small}
\end{table}
)LaTeX";

enum riscv_latex_type {
	riscv_latex_type_empty,
	riscv_latex_type_line,
	riscv_latex_type_page_break,
	riscv_latex_type_extension_heading,
	riscv_latex_type_extension_contd,
	riscv_latex_type_opcode,
	riscv_latex_type_type_bitrange,
	riscv_latex_type_type_spec,
};

struct riscv_latex_row
{
	riscv_latex_type row_type;
	riscv_extension_ptr extension;
	riscv_opcode_ptr opcode;
	riscv_type_ptr type;

	riscv_latex_row(riscv_latex_type row_type)
		: row_type(row_type) {}

	riscv_latex_row(riscv_latex_type row_type, riscv_extension_ptr extension)
		: row_type(row_type), extension(extension) {}

	riscv_latex_row(riscv_latex_type row_type, riscv_opcode_ptr opcode)
		: row_type(row_type), opcode(opcode) {}

	riscv_latex_row(riscv_latex_type row_type, riscv_type_ptr type)
		: row_type(row_type), type(type) {}
};

struct riscv_latex_page
{
	std::vector<riscv_latex_row> rows;
};

struct riscv_codec_node
{
	std::vector<ssize_t> bits;
	std::vector<ssize_t> vals;
	std::map<ssize_t,riscv_opcode_list> val_opcodes;
	std::map<ssize_t,riscv_codec_node> val_decodes;

	void clear();
};

struct riscv_parse_meta : riscv_meta_model
{
	riscv_extension_list     ext_subset;
	riscv_codec_node         root_node;
	riscv_opcode_ptr         unknown;

	void generate_map();
	void generate_codec(bool include_pseudo);

	std::string colorize_args(riscv_opcode_ptr opcode);
	std::vector<std::string> get_unique_codecs();

	static std::string latex_utf_substitute(std::string str);

	void print_latex_row(riscv_latex_row &row, std::string ts);
	void print_latex();
	void print_map(bool print_map_instructions);
	void print_c_header(std::string filename);
	void print_opcodes_h(bool no_comment = false, bool zero_not_oh = false);
	void print_opcodes_c(bool no_comment = false, bool zero_not_oh = false);
	void print_args_h();
	void print_codec_h(bool no_comment = false, bool zero_not_oh = false);

	void generate_codec_node(riscv_codec_node &node, riscv_opcode_list &opcode_list);
	void print_switch_decoder_node(riscv_codec_node &node, size_t indent);
};

template <typename T>
std::string join(std::vector<T> list, std::string sep)
{
	std::stringstream ss;
	for (auto i = list.begin(); i != list.end(); i++) {
		ss << (i != list.begin() ? sep : "") << *i;
	}
	return ss.str();
}

void riscv_codec_node::clear()
{
	bits.clear();
	vals.clear();
	val_opcodes.clear();
	val_decodes.clear();
}

void riscv_parse_meta::generate_map()
{
	for (auto &opcode : opcodes) {
		for (auto &mask : opcode->masks) {
			ssize_t msb = mask.first.msb;
			ssize_t lsb = mask.first.lsb;
			ssize_t val = mask.second;
			for (ssize_t bit = msb; bit >= lsb; bit--) {
				opcode->mask |= (1 << bit);
				opcode->match |= (val >> (bit - lsb)) & 1 ? (1 << bit) : 0;
			}
		}
	}
}

void riscv_parse_meta::generate_codec(bool include_pseudo)
{
	// make list of opcodes to include
	riscv_opcode_list opcodes_copy;
	for (auto &opcode : opcodes) {
		if (include_pseudo || opcode->key.find("@") == std::string::npos) {
			opcodes_copy.push_back(opcode);
		}
	}

	// generate decode
	root_node.clear();
	generate_codec_node(root_node, opcodes_copy);
}

std::string riscv_parse_meta::colorize_args(riscv_opcode_ptr opcode)
{
	std::vector<char> token;
	std::vector<std::string> comps;
	auto args = opcode->format->args;

	for (size_t i = 0; i < args.length(); i++) {
		char c = args[i];
		if (::isalnum(c)) {
			token.push_back(c);
		} else {
			comps.push_back(std::string(token.begin(), token.end()));
			token.resize(0);
			token.push_back(c);
			comps.push_back(std::string(token.begin(), token.end()));
			token.resize(0);
		}
	}
	if (token.size() > 0) {
		comps.push_back(std::string(token.begin(), token.end()));
	}

	for (size_t i = 0; i < comps.size(); i++) {
		auto comp = comps[i];
		auto arg = args_by_name[comp];
		if (arg) {
			auto new_comp = riscv_colors_to_ansi_escape_sequence(
				arg->fg_color, arg->bg_color, ansi_color_normal
			);
			new_comp.append(comp);
			new_comp.append(_COLOR_RESET);
			comps[i] = new_comp;
		}
	}

	return join(comps, "");
}

std::vector<std::string> riscv_parse_meta::get_unique_codecs()
{
	std::vector<std::string> codec_names;
	for (auto &codec : codecs) {
		std::string codec_name = format_codec("", codec, "_");
		if (std::find(codec_names.begin(), codec_names.end(), codec_name) == codec_names.end()) {
			codec_names.push_back(codec_name);
		}
	}
	return codec_names;
}

std::string riscv_parse_meta::latex_utf_substitute(std::string str)
{
	str = replace(str, "∞", "\\infty");
	str = replace(str, "{", "\\left\\{");
	str = replace(str, "}", "\\right\\}");
	str = replace(str, "≠", "\\neq");
	str = replace(str, "≤", "\\leq");
	str = replace(str, "≥", "\\geq");
	str = replace(str, "⊂", "\\subset");
	str = replace(str, "⊃", "\\supset");
	str = replace(str, "⊆", "\\subseteq");
	str = replace(str, "⊇", "\\supseteq");
	str = replace(str, "⊕", "\\oplus");
	str = replace(str, "⊗", "\\otimes");
	str = replace(str, "⊖", "\\ominus");
	str = replace(str, "→", "\\rightarrow");
	str = replace(str, "←", "\\leftarrow");
	str = replace(str, "∨", "\\vee");
	str = replace(str, "∧", "\\wedge");
	str = replace(str, "⊻", "\\veebar");
	str = replace(str, "⊼", "\\wedgebar");
	str = replace(str, "×", "\\times");
	str = replace(str, "÷", "\\div");
	str = replace(str, "«", "\\ll");
	str = replace(str, "»", "\\gg");
	str = replace(str, "∥", "\\parallel");
	return str;
}

void riscv_parse_meta::print_latex_row(riscv_latex_row &row, std::string ts)
{
	switch (row.row_type) {
		case riscv_latex_type_empty:
			printf("\\tabularnewline\n");
			break;
		case riscv_latex_type_line:
			printf("\\cline{1-%ld}\n", kLatexTableColumns);
			break;
		case riscv_latex_type_page_break:
			printf("%s", kLatexTableEnd);
			printf("%s%s", kLatexTableBegin, ts.c_str());
			break;
		case riscv_latex_type_extension_heading:
		case riscv_latex_type_extension_contd:
		{
			printf("\\multicolumn{%ld}{c}{\\bf %s %s} & \\\\\n\\cline{1-%ld}\n",
				kLatexTableColumns,
				row.extension->description.c_str(),
				row.row_type == riscv_latex_type_extension_contd ? " contd" : "",
				kLatexTableColumns);
			break;
		}
		case riscv_latex_type_opcode:
{'fraction_non_alphanumeric': 0.14921574023115025, 'fraction_numerical': 0.0029581728123280132, 'mean_word_length': 5.320434782608696, 'pattern_counts': {'":': 0, '<': 81, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1841455', 'n_tokens_mistral': 6069, 'n_tokens_neox': 5850, 'n_words': 1063}