<commit_before>// SciTE - Scintilla based Text Editor
/** @file ExportRTF.cxx
 ** Export the current document to RTF.
 **/
// Copyright 1998-2006 by <NAME> <<EMAIL>>
// The License.txt file describes the conditions under which this software may be distributed.

#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <stdio.h>
#include <stddef.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <time.h>

#include <string>
#include <vector>
#include <set>
#include <map>
#include <sstream>

#include "Scintilla.h"
#include "ILexer.h"

#include "GUI.h"

#include "StringList.h"
#include "StringHelpers.h"
#include "FilePath.h"
#include "StyleDefinition.h"
#include "PropSetFile.h"
#include "StyleWriter.h"
#include "Extender.h"
#include "SciTE.h"
#include "Mutex.h"
#include "JobQueue.h"
#include "Cookie.h"
#include "Worker.h"
#include "MatchMarker.h"
#include "SciTEBase.h"


//---------- Save to RTF ----------

#define RTF_HEADEROPEN "{\\rtf1\\ansi\\deff0\\deftab720"
#define RTF_FONTDEFOPEN "{\\fonttbl"
#define RTF_FONTDEFCLOSE "}"
#define RTF_COLORDEFOPEN "{\\colortbl"
#define RTF_COLORDEFCLOSE "}"
#define RTF_HEADERCLOSE "\n"
#define RTF_BODYOPEN ""
#define RTF_BODYCLOSE "}"

#define RTF_SETFONTFACE "\\f"
#define RTF_SETFONTSIZE "\\fs"
#define RTF_SETCOLOR "\\cf"
#define RTF_SETBACKGROUND "\\highlight"
#define RTF_BOLD_ON "\\b"
#define RTF_BOLD_OFF "\\b0"
#define RTF_ITALIC_ON "\\i"
#define RTF_ITALIC_OFF "\\i0"
#define RTF_UNDERLINE_ON "\\ul"
#define RTF_UNDERLINE_OFF "\\ulnone"
#define RTF_STRIKE_ON "\\i"
#define RTF_STRIKE_OFF "\\strike0"

#define RTF_EOLN "\\par\n"
#define RTF_TAB "\\tab "

#define MAX_STYLEDEF 128
#define MAX_FONTDEF 64
#define MAX_COLORDEF 8
#define RTF_FONTFACE "Courier New"
#define RTF_COLOR "#000000"

// extract the next RTF control word from *style
void GetRTFNextControl(char **style, char *control) {
	ptrdiff_t len;
	char *pos = *style;
	*control = '\0';
	if ('\0' == *pos) return;
	pos++; // implicit skip over leading '\'
	while ('\0' != *pos && '\\' != *pos) { pos++; }
	len = pos - *style;
	memcpy(control, *style, len);
	*(control + len) = '\0';
	*style = pos;
}

// extracts control words that are different between two styles
void GetRTFStyleChange(char *delta, char *last, char *current) { // \f0\fs20\cf0\highlight0\b0\i0
	char lastControl[MAX_STYLEDEF], currentControl[MAX_STYLEDEF];
	char *lastPos = last;
	char *currentPos = current;
	*delta = '\0';
	// font face, size, color, background, bold, italic
	for (int i = 0; i < 6; i++) {
		GetRTFNextControl(&lastPos, lastControl);
		GetRTFNextControl(&currentPos, currentControl);
		if (strcmp(lastControl, currentControl)) {	// changed
			strcat(delta, currentControl);
		}
	}
	if ('\0' != *delta) { strcat(delta, " "); }
	strcpy(last, current);
}

void SciTEBase::SaveToStreamRTF(std::ostream &os, int start, int end) {
	int lengthDoc = LengthDocument();
	if (end < 0)
		end = lengthDoc;
	RemoveFindMarks();
	wEditor.Call(SCI_COLOURISE, 0, -1);

	StyleDefinition defaultStyle = StyleDefinitionFor(STYLE_DEFAULT);

	int tabSize = props.GetInt("export.rtf.tabsize", props.GetInt("tabsize"));
	int wysiwyg = props.GetInt("export.rtf.wysiwyg", 1);
	std::string fontFace = props.GetExpandedString("export.rtf.font.face");
	if (fontFace.length()) {
		defaultStyle.font = fontFace;
	} else if (defaultStyle.font.length() == 0) {
		defaultStyle.font = RTF_FONTFACE;
	}
	int fontSize = props.GetInt("export.rtf.font.size", 0);
	if (fontSize > 0) {
		defaultStyle.size = fontSize << 1;
	} else if (defaultStyle.size == 0) {
		defaultStyle.size = 10 << 1;
	} else {
		defaultStyle.size <<= 1;
	}
	unsigned int characterset = props.GetInt("character.set", SC_CHARSET_DEFAULT);
	int tabs = props.GetInt("export.rtf.tabs", 0);
	if (tabSize == 0)
		tabSize = 4;

	char styles[STYLE_MAX + 1][MAX_STYLEDEF];
	char fonts[STYLE_MAX + 1][MAX_FONTDEF];
	char colors[STYLE_MAX + 1][MAX_COLORDEF];
	char lastStyle[MAX_STYLEDEF], deltaStyle[MAX_STYLEDEF];
	int fontCount = 1, colorCount = 2, i;
	os << RTF_HEADEROPEN << RTF_FONTDEFOPEN;
	StringCopy(fonts[0], defaultStyle.font.c_str());
	os << "{\\f" << 0 << "\\fnil\\fcharset" << characterset << " " << defaultStyle.font.c_str() << ";}";
	StringCopy(colors[0], defaultStyle.fore.c_str());
	StringCopy(colors[1], defaultStyle.back.c_str());

	for (int istyle = 0; istyle <= STYLE_MAX; istyle++) {

		StyleDefinition sd = StyleDefinitionFor(istyle);

		if (sd.specified != StyleDefinition::sdNone) {
			if (wysiwyg && sd.font.length()) {
				for (i = 0; i < fontCount; i++)
					if (EqualCaseInsensitive(sd.font.c_str(), fonts[i]))
						break;
				if (i >= fontCount) {
					StringCopy(fonts[fontCount++], sd.font.c_str());
					os << "{\\f" << i << "\\fnil\\fcharset" << characterset << " " << sd.font.c_str() << ";}";
				}
				sprintf(lastStyle, RTF_SETFONTFACE "%d", i);
			} else {
				strcpy(lastStyle, RTF_SETFONTFACE "0");
			}

			sprintf(lastStyle + strlen(lastStyle), RTF_SETFONTSIZE "%d",
				wysiwyg && sd.size ? sd.size << 1 : defaultStyle.size);

			if (sd.specified & StyleDefinition::sdFore) {
				for (i = 0; i < colorCount; i++)
					if (EqualCaseInsensitive(sd.fore.c_str(), colors[i]))
						break;
				if (i >= colorCount)
					StringCopy(colors[colorCount++], sd.fore.c_str());
				sprintf(lastStyle + strlen(lastStyle), RTF_SETCOLOR "%d", i);
			} else {
				strcat(lastStyle, RTF_SETCOLOR "0");	// Default fore
			}

			// PL: highlights doesn't seems to follow a distinct table, at least with WordPad and Word 97
			// Perhaps it is different for Word 6?
//				sprintf(lastStyle + strlen(lastStyle), RTF_SETBACKGROUND "%d",
//				        sd.back.length() ? GetRTFHighlight(sd.back.c_str()) : 0);
			if (sd.specified & StyleDefinition::sdBack) {
				for (i = 0; i < colorCount; i++)
					if (EqualCaseInsensitive(sd.back.c_str(), colors[i]))
						break;
				if (i >= colorCount)
					StringCopy(colors[colorCount++], sd.back.c_str());
				sprintf(lastStyle + strlen(lastStyle), RTF_SETBACKGROUND "%d", i);
			} else {
				strcat(lastStyle, RTF_SETBACKGROUND "1");	// Default back
			}
			if (sd.specified & StyleDefinition::sdWeight) {
				strcat(lastStyle, sd.IsBold() ? RTF_BOLD_ON : RTF_BOLD_OFF);
			} else {
				strcat(lastStyle, defaultStyle.IsBold() ? RTF_BOLD_ON : RTF_BOLD_OFF);
			}
			if (sd.specified & StyleDefinition::sdItalics) {
				strcat(lastStyle, sd.italics ? RTF_ITALIC_ON : RTF_ITALIC_OFF);
			} else {
				strcat(lastStyle, defaultStyle.italics ? RTF_ITALIC_ON : RTF_ITALIC_OFF);
			}
			StringCopy(styles[istyle], lastStyle);
		} else {
			sprintf(styles[istyle], RTF_SETFONTFACE "0" RTF_SETFONTSIZE "%d"
				RTF_SETCOLOR "0" RTF_SETBACKGROUND "1"
				RTF_BOLD_OFF RTF_ITALIC_OFF, defaultStyle.size);
		}
	}
	os << RTF_FONTDEFCLOSE RTF_COLORDEFOPEN;
	for (i = 0; i < colorCount; i++) {
		os << "\\red" << IntFromHexByte(colors[i] + 1) << "\\green" << IntFromHexByte(colors[i] + 3) <<
			"\\blue" << IntFromHexByte(colors[i] + 5) << ";";
	}
	os << RTF_COLORDEFCLOSE RTF_HEADERCLOSE RTF_BODYOPEN RTF_SETFONTFACE "0"
		RTF_SETFONTSIZE << defaultStyle.size << RTF_SETCOLOR "0 ";
	sprintf(lastStyle, RTF_SETFONTFACE "0" RTF_SETFONTSIZE "%d"
		RTF_SETCOLOR "0" RTF_SETBACKGROUND "1"
		RTF_BOLD_OFF RTF_ITALIC_OFF, defaultStyle.size);
	bool prevCR = false;
	int styleCurrent = -1;
	TextReader acc(wEditor);
	int column = 0;
	for (i = start; i < end; i++) {
		char ch = acc[i];
		int style = acc.StyleAt(i);
		if (style > STYLE_MAX)
			style = 0;
		if (style != styleCurrent) {
			GetRTFStyleChange(deltaStyle, lastStyle, styles[style]);
			if (*deltaStyle)
				os << deltaStyle;
			styleCurrent = style;
		}
		if (ch == '{')
			os << "\\{";
		else if (ch == '}')
			os << "\\}";
		else if (ch == '\\')
			os << "\\\\";
		else if (ch == '\t') {
			if (tabs) {
				os << RTF_TAB;
			} else {
				int ts = tabSize - (column % tabSize);
				for (int itab = 0; itab < ts; itab++) {
					os << ' ';
				}
				column += ts - 1;
			}<commit_msg>Use dynamic allocation to simplify code and remove small possibility of writing outside allocations.<commit_after>// SciTE - Scintilla based Text Editor
/** @file ExportRTF.cxx
 ** Export the current document to RTF.
 **/
// Copyright 1998-2006 by <NAME> <<EMAIL>>
// The License.txt file describes the conditions under which this software may be distributed.

#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include <stdio.h>
#include <stddef.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <time.h>

#include <string>
#include <vector>
#include <set>
#include <map>
#include <sstream>

#include "Scintilla.h"
#include "ILexer.h"

#include "GUI.h"

#include "StringList.h"
#include "StringHelpers.h"
#include "FilePath.h"
#include "StyleDefinition.h"
#include "PropSetFile.h"
#include "StyleWriter.h"
#include "Extender.h"
#include "SciTE.h"
#include "Mutex.h"
#include "JobQueue.h"
#include "Cookie.h"
#include "Worker.h"
#include "MatchMarker.h"
#include "SciTEBase.h"


//---------- Save to RTF ----------

#define RTF_HEADEROPEN "{\\rtf1\\ansi\\deff0\\deftab720"
#define RTF_FONTDEFOPEN "{\\fonttbl"
#define RTF_FONTDEFCLOSE "}"
#define RTF_COLORDEFOPEN "{\\colortbl"
#define RTF_COLORDEFCLOSE "}"
#define RTF_HEADERCLOSE "\n"
#define RTF_BODYOPEN ""
#define RTF_BODYCLOSE "}"

#define RTF_SETFONTFACE "\\f"
#define RTF_SETFONTSIZE "\\fs"
#define RTF_SETCOLOR "\\cf"
#define RTF_SETBACKGROUND "\\highlight"
#define RTF_BOLD_ON "\\b"
#define RTF_BOLD_OFF "\\b0"
#define RTF_ITALIC_ON "\\i"
#define RTF_ITALIC_OFF "\\i0"
#define RTF_UNDERLINE_ON "\\ul"
#define RTF_UNDERLINE_OFF "\\ulnone"
#define RTF_STRIKE_ON "\\i"
#define RTF_STRIKE_OFF "\\strike0"

#define RTF_EOLN "\\par\n"
#define RTF_TAB "\\tab "

#define MAX_STYLEDEF 128
#define RTF_FONTFACE "Courier New"
#define RTF_COLOR "#000000"

static size_t FindCaseInsensitive(const std::vector<std::string> &values, const std::string &s) {
	for (size_t i = 0; i < values.size(); i++)
		if (EqualCaseInsensitive(s.c_str(), values[i].c_str()))
			return i;
	return values.size();
}

// extract the next RTF control word from *style
void GetRTFNextControl(const char **style, char *control) {
	ptrdiff_t len;
	const char *pos = *style;
	*control = '\0';
	if ('\0' == *pos) return;
	pos++; // implicit skip over leading '\'
	while ('\0' != *pos && '\\' != *pos) { pos++; }
	len = pos - *style;
	memcpy(control, *style, len);
	*(control + len) = '\0';
	*style = pos;
}

// extracts control words that are different between two styles
std::string GetRTFStyleChange(const char *last, const char *current) { // \f0\fs20\cf0\highlight0\b0\i0
	char lastControl[MAX_STYLEDEF], currentControl[MAX_STYLEDEF];
	const char *lastPos = last;
	const char *currentPos = current;
	std::string delta;
	// font face, size, color, background, bold, italic
	for (int i = 0; i < 6; i++) {
		GetRTFNextControl(&lastPos, lastControl);
		GetRTFNextControl(&currentPos, currentControl);
		if (strcmp(lastControl, currentControl)) {	// changed
			delta += currentControl;
		}
	}
	if (!delta.empty()) { delta += " "; }
	return delta;
}

void SciTEBase::SaveToStreamRTF(std::ostream &os, int start, int end) {
	int lengthDoc = LengthDocument();
	if (end < 0)
		end = lengthDoc;
	RemoveFindMarks();
	wEditor.Call(SCI_COLOURISE, 0, -1);

	StyleDefinition defaultStyle = StyleDefinitionFor(STYLE_DEFAULT);

	int tabSize = props.GetInt("export.rtf.tabsize", props.GetInt("tabsize"));
	int wysiwyg = props.GetInt("export.rtf.wysiwyg", 1);
	std::string fontFace = props.GetExpandedString("export.rtf.font.face");
	if (fontFace.length()) {
		defaultStyle.font = fontFace;
	} else if (defaultStyle.font.length() == 0) {
		defaultStyle.font = RTF_FONTFACE;
	}
	int fontSize = props.GetInt("export.rtf.font.size", 0);
	if (fontSize > 0) {
		defaultStyle.size = fontSize << 1;
	} else if (defaultStyle.size == 0) {
		defaultStyle.size = 10 << 1;
	} else {
		defaultStyle.size <<= 1;
	}
	unsigned int characterset = props.GetInt("character.set", SC_CHARSET_DEFAULT);
	int tabs = props.GetInt("export.rtf.tabs", 0);
	if (tabSize == 0)
		tabSize = 4;

	std::vector<std::string> styles;
	std::vector<std::string> fonts;
	std::vector<std::string> colors;
	os << RTF_HEADEROPEN << RTF_FONTDEFOPEN;
	fonts.push_back(defaultStyle.font);
	os << "{\\f" << 0 << "\\fnil\\fcharset" << characterset << " " << defaultStyle.font.c_str() << ";}";
	colors.push_back(defaultStyle.fore);
	colors.push_back(defaultStyle.back);

	for (int istyle = 0; istyle <= STYLE_MAX; istyle++) {
		std::ostringstream osStyle;

		StyleDefinition sd = StyleDefinitionFor(istyle);

		if (sd.specified != StyleDefinition::sdNone) {
			size_t iFont = 0;
			if (wysiwyg && sd.font.length()) {
				iFont = FindCaseInsensitive(fonts, sd.font);
				if (iFont >= fonts.size()) {
					fonts.push_back(sd.font);
					os << "{\\f" << iFont << "\\fnil\\fcharset" << characterset << " " << sd.font.c_str() << ";}";
				}
			}
			osStyle << RTF_SETFONTFACE << iFont;

			osStyle << RTF_SETFONTSIZE << (wysiwyg && sd.size ? sd.size << 1 : defaultStyle.size);

			size_t iFore = 0;
			if (sd.specified & StyleDefinition::sdFore) {
				iFore = FindCaseInsensitive(colors, sd.fore);
				if (iFore >= colors.size())
					colors.push_back(sd.fore);
			}
			osStyle << RTF_SETCOLOR << iFore;

			// PL: highlights doesn't seems to follow a distinct table, at least with WordPad and Word 97
			// Perhaps it is different for Word 6?
			size_t iBack = 1;
			if (sd.specified & StyleDefinition::sdBack) {
				iBack = FindCaseInsensitive(colors, sd.back);
				if (iBack >= colors.size())
					colors.push_back(sd.back);
			}
			osStyle << RTF_SETBACKGROUND << iBack;

			if (sd.specified & StyleDefinition::sdWeight) {
				osStyle << (sd.IsBold() ? RTF_BOLD_ON : RTF_BOLD_OFF);
			} else {
				osStyle << (defaultStyle.IsBold() ? RTF_BOLD_ON : RTF_BOLD_OFF);
			}
			if (sd.specified & StyleDefinition::sdItalics) {
				osStyle << (sd.italics ? RTF_ITALIC_ON : RTF_ITALIC_OFF);
			} else {
				osStyle << (defaultStyle.italics ? RTF_ITALIC_ON : RTF_ITALIC_OFF);
			}
		} else {
			osStyle << RTF_SETFONTFACE "0" RTF_SETFONTSIZE << defaultStyle.size <<
			        RTF_SETCOLOR "0" RTF_SETBACKGROUND "1"
			        RTF_BOLD_OFF RTF_ITALIC_OFF;
		}
		styles.push_back(osStyle.str());
	}
	os << RTF_FONTDEFCLOSE RTF_COLORDEFOPEN;
	for (size_t i = 0; i < colors.size(); i++) {
		os << "\\red" << IntFromHexByte(colors[i].c_str() + 1) << "\\green" << IntFromHexByte(colors[i].c_str() + 3) <<
		   "\\blue" << IntFromHexByte(colors[i].c_str() + 5) << ";";
	}
	os << RTF_COLORDEFCLOSE RTF_HEADERCLOSE RTF_BODYOPEN RTF_SETFONTFACE "0"
	   RTF_SETFONTSIZE << defaultStyle.size << RTF_SETCOLOR "0 ";
	std::ostringstream osStyleDefault;
	osStyleDefault << RTF_SETFONTFACE "0" RTF_SETFONTSIZE << defaultStyle.size <<
	               RTF_SETCOLOR "0" RTF_SETBACKGROUND "1"
	               RTF_BOLD_OFF RTF_ITALIC_OFF;
	std::string lastStyle = osStyleDefault.str();
	bool prevCR = false;
	int styleCurrent = -1;
	TextReader acc(wEditor);
	int column = 0;
	for (int iPos = start; iPos < end; iPos++) {
		char ch = acc[iPos];
		int style = acc.StyleAt(iPos);
		if (style > STYLE_MAX)
			style = 0;
		if (style != styleCurrent) {
			const std::string deltaStyle = GetRTFStyleChange(lastStyle.c_str(), styles[style].c_str());
			lastStyle = styles[style];
			if (!deltaStyle.empty())
				os << deltaStyle;
			styleCurrent = style;
		}
		if (ch == '{')
			os << "\\{";
		else if (ch == '}')
			os << "\\}";
		else if (ch == '\\')
			os << "\\\\";
		else if (ch == '\t') {
			if (tabs) {
				os << RTF_TAB;
			} else {
				int ts = tabSize - (column % tabSize);
				for (int itab = 0; itab < ts; itab++) {
					os << ' ';
				}
				column += ts - 1;
			}
{'fraction_non_alphanumeric': 0.16735946792863082, 'fraction_numerical': 0.011383257658118566, 'mean_word_length': 3.930012610340479, 'pattern_counts': {'":': 0, '<': 239, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7345696', 'n_tokens_mistral': 6582, 'n_tokens_neox': 5886, 'n_words': 1350}