<commit_before>#!/usr/bin/perl -w

# convert plain ascii to Yacas manual code
# Syntax of ascii files is described in "txt2yacasdoc" essay
#
# Use with care, test output with manualmaker

$debug_pars = ("@ARGV" =~ /-debug/) ? 1 : 0;	# This is for debugging only: breaks HTML docs, but it is useful because it makes all paragraphs into separate Text() invocations and then it's a lot easier to debug book2TeX or manualmaker in Yacas - just look at the resulting .tex or .book file to see where it stopped generating.

$have_Text = 0;	# Means we have a Text() declared somewhere above
$have_par = 1;	# Means we already have a new paragraph
$in_quotes = 0;	# Means we are inside quotes of the Text()
$in_itemized = 0;
$in_enum = 0;
$in_htmlcommand = 0;

%star_labels = ( 	# without parameters
	"CALL" => "Topical() \"Calling format:\";",
	"PARMS" => "Topical() \"Parameters:\";",
	"DESC" => "Topical() \"Description:\";",
	"UNIX" => "UnixSpecific();",
	"MSWIN" => "MSWinSpecific();",
	"MAC" => "MacSpecific();",
	"STD" => "StandardLib();",
	"CORE" => "BuiltIn();",
	"E.G." => "Topical() \"Examples:\";",
	"EG" => "Topical() \"Example:\";",
);

%intro_labels = (
	"BLURB" => "Blurb()",
	"INTRO" => "ChapterIntro()",
	"REM" => "DocumentationComment()",
);

while (<STDIN>) {
	chomp;
	s/\\/\\\\/g;	# escape all backslashes in the source text
	s/"/\\"/g;	# escape all quotes in the source text
	# First deal with TAB-indented lines
	if (not $in_htmlcommand and /^\t\t\t\t([^ ].*)$/) {	# Book
		&finish_text_close_quote();
		print "\tBook()\"", &escape_term($1), "\";\n\n";
		$have_par = 1;
	} elsif (not $in_htmlcommand and /^\t\t\t([^ ].*)$/) {	# Chapter
		&finish_text_close_quote();
		print "\tChapter()\"", &escape_term($1), "\";\n\n";
		$have_par = 1;
	} elsif (not $in_htmlcommand and /^\t\t([^ ].*)$/) {	# Section
		&finish_text_close_quote();
		print "\tSection()\"", &escape_term($1), "\";\n\n";
		$have_par = 1;
	} elsif (not $in_htmlcommand and /^\t    ([^ ].*)$/) {	# Subsection
		&finish_text_close_quote();
		print "\tSubSection()\"", &escape_term($1), "\";\n\n";
		$have_par = 1;
	}
	#############################################################
	# HtmlCommand processed here
	#############################################################
	elsif ((not $in_htmlcommand and /^\t( {0,2}[^ \t].*)$/) or ($in_htmlcommand and /^\t(.*)$/)) {	# Sample code (HtmlCommand); cannot start with empty line
		&start_text_open_quote();
		if (not $in_htmlcommand) {
			if ($debug_pars) {
				&finish_text_close_quote();
				&start_text_open_quote();
			}
			&close_quote();
			print ":HtmlCommand(\"\n";
			$in_htmlcommand = 1;
		}
		print "$1\n" unless ($1 eq "*");	# Special feature to allow code examples that start with indented lines
		$have_par = 0;
		$in_quotes = 1;
	}	# Now processing not-TAB-indented lines
	#############################################################
	# New paragraph	
	#############################################################
	elsif (/^\s*$/) {
		if ($have_par == 0) {
			&finish_text_close_quote();
			&start_text_open_quote();
			print "\":HtmlNewParagraph():\n\n\"";
			$have_par = 1;
		}
	}
	#############################################################
	#			Star-based markup
	#############################################################
	elsif (/^\*\t[0-9]+\. (.*)$/) {	# Enum
			if ($debug_pars) {
				&finish_text_close_quote();
				&start_text_open_quote();
			}
		&start_text_open_quote();
		&close_quote();
		if ($in_enum) {
			print ":Item()\"", &escape_term($1), "\n";
		} else {
			print ":Enumerate() (\nItem()\"", &escape_term($1), "\n";
			$in_enum = 1;
		}
		$in_quotes = 1;
		$have_par = 0;
	} elsif (/^\*\t(.*)$/) {	# Itemized
			if ($debug_pars) {
				&finish_text_close_quote();
				&start_text_open_quote();
			}
		&start_text_open_quote();
		&close_quote();
		if ($in_itemized) {
			print ":Item()\"", &escape_term($1), "\n";
		} else {
			print ":Itemize() (\nItem()\"", &escape_term($1), "\n";
			$in_itemized = 1;
		}
		$in_quotes = 1;
		$have_par = 0;
	}	# Now non-TAB indented markup
	elsif (/^\*INCLUDE\s\s*([^ ].*)\s*$/i) {	# Include another file
		&finish_text_close_quote();
		$have_par = 1;
		print "IncludeFile(\"$1\");\n\n";
	}
	elsif (/^\*EVAL\s\s*([^ ].*)\s*$/i) {	# Evaluate a Yacas expression and insert its result as text
		$expression = $1;
		# un-escape quotes and backslash again
		$expression =~ s/\\"/"/g;
		$expression =~ s/\\\\/\\/g;
		&finish_text_close_quote();
		$have_par = 0;	# even if we had a paragraph before us, now we don't
		print "DocEval($expression);\n";	# this will insert inline Yacas code
	} elsif (/^\*FOOT\s\s*(.*)$/) {	# footnote
		&finish_text_close_quote();
		#$have_par = 0;
		print "AddBody(DocFootnote(\"" . $1 . "\"));\n";
	}
	#############################################################
	# stuff for refman
	#############################################################
	elsif (/^\*([-A-Z.]+)\s*$/ and defined($star_labels{$1})) {	# labels without parameters
		&finish_text_close_quote();
		$have_par = 1;
		print $star_labels{$1} . "\n";
	} elsif (/^\*(?:HEAD)\s\s*(.*)$/) {	# Topical()
		&finish_text_close_quote();
		$have_par = 1;
		print "Topical()\"" . &escape_term($1) . "\";\n";
	} elsif (/^\*A\s\s*(.*)$/) {	# anchor
		&finish_text_close_quote();
		#$have_par = 1;
		$anchor = $1;
		# if the first word is {...}, then we need to use the @ stuff
		$anchor =~ s/^\{([^{}]+)\}/$1 . "@" . &escape_term(
"{$1}")/e;
		print "AddBody(AddAnchor(\"" . $anchor . "\"));\n";
	} elsif (/^\*SEE\s\s*(.*)$/) {	# SeeAlso()
		$names = $1;
		$names =~ s/\s*$//;
		$names =~ s/^\s*//;
		&finish_text_close_quote();
		$have_par = 1;
		print "Topical()\"See also:\";\nSeeAlso({";
		$have_prev_name = 0;
		foreach $name (split(/,[ \t]+/, $names)) {
			print ((($have_prev_name == 1) ? ", " : "") . "\"$name\"");
			$have_prev_name = 1;
		}
		print "});\n";
	} elsif (/^\*SEE\s*(.*)$/) {	# empty SeeAlso()
		&finish_text_close_quote();
		$have_par = 1;
		# do nothing
	} elsif (/^\*(?:CMD|FUNC)\s\s*(.*)\s*---\s*(.*)$/) {	# CmdDescription()
		$names = $1;
		$title = $2;
		&finish_text_close_quote();
		$have_par = 1;
		print "\n";	# separate from previous command (has no effect on docs)
		# Trim extra spaces
		$names =~ s/\s*$//;<commit_msg>bug fix: TeX code in footnotes and anchors was not processed<commit_after>#!/usr/bin/perl -w

# convert plain ascii to Yacas manual code
# Syntax of ascii files is described in "txt2yacasdoc" essay
#
# Use with care, test output with manualmaker

$debug_pars = ("@ARGV" =~ /-debug/) ? 1 : 0;	# This is for debugging only: breaks HTML docs, but it is useful because it makes all paragraphs into separate Text() invocations and then it's a lot easier to debug book2TeX or manualmaker in Yacas - just look at the resulting .tex or .book file to see where it stopped generating.

$have_Text = 0;	# Means we have a Text() declared somewhere above
$have_par = 1;	# Means we already have a new paragraph
$in_quotes = 0;	# Means we are inside quotes of the Text()
$in_itemized = 0;
$in_enum = 0;
$in_htmlcommand = 0;

%star_labels = ( 	# without parameters
	"CALL" => "Topical() \"Calling format:\";",
	"PARMS" => "Topical() \"Parameters:\";",
	"DESC" => "Topical() \"Description:\";",
	"UNIX" => "UnixSpecific();",
	"MSWIN" => "MSWinSpecific();",
	"MAC" => "MacSpecific();",
	"STD" => "StandardLib();",
	"CORE" => "BuiltIn();",
	"E.G." => "Topical() \"Examples:\";",
	"EG" => "Topical() \"Example:\";",
);

%intro_labels = (
	"BLURB" => "Blurb()",
	"INTRO" => "ChapterIntro()",
	"REM" => "DocumentationComment()",
);

while (<STDIN>) {
	chomp;
	s/\\/\\\\/g;	# escape all backslashes in the source text
	s/"/\\"/g;	# escape all quotes in the source text
	# First deal with TAB-indented lines
	if (not $in_htmlcommand and /^\t\t\t\t([^ ].*)$/) {	# Book
		&finish_text_close_quote();
		print "\tBook()\"", &escape_term($1), "\";\n\n";
		$have_par = 1;
	} elsif (not $in_htmlcommand and /^\t\t\t([^ ].*)$/) {	# Chapter
		&finish_text_close_quote();
		print "\tChapter()\"", &escape_term($1), "\";\n\n";
		$have_par = 1;
	} elsif (not $in_htmlcommand and /^\t\t([^ ].*)$/) {	# Section
		&finish_text_close_quote();
		print "\tSection()\"", &escape_term($1), "\";\n\n";
		$have_par = 1;
	} elsif (not $in_htmlcommand and /^\t    ([^ ].*)$/) {	# Subsection
		&finish_text_close_quote();
		print "\tSubSection()\"", &escape_term($1), "\";\n\n";
		$have_par = 1;
	}
	#############################################################
	# HtmlCommand processed here
	#############################################################
	elsif ((not $in_htmlcommand and /^\t( {0,2}[^ \t].*)$/) or ($in_htmlcommand and /^\t(.*)$/)) {	# Sample code (HtmlCommand); cannot start with empty line
		&start_text_open_quote();
		if (not $in_htmlcommand) {
			if ($debug_pars) {
				&finish_text_close_quote();
				&start_text_open_quote();
			}
			&close_quote();
			print ":HtmlCommand(\"\n";
			$in_htmlcommand = 1;
		}
		print "$1\n" unless ($1 eq "*");	# Special feature to allow code examples that start with indented lines
		$have_par = 0;
		$in_quotes = 1;
	}	# Now processing not-TAB-indented lines
	#############################################################
	# New paragraph	
	#############################################################
	elsif (/^\s*$/) {
		if ($have_par == 0) {
			&finish_text_close_quote();
			&start_text_open_quote();
			print "\":HtmlNewParagraph():\n\n\"";
			$have_par = 1;
		}
	}
	#############################################################
	#			Star-based markup
	#############################################################
	elsif (/^\*\t[0-9]+\. (.*)$/) {	# Enum
			if ($debug_pars) {
				&finish_text_close_quote();
				&start_text_open_quote();
			}
		&start_text_open_quote();
		&close_quote();
		if ($in_enum) {
			print ":Item()\"", &escape_term($1), "\n";
		} else {
			print ":Enumerate() (\nItem()\"", &escape_term($1), "\n";
			$in_enum = 1;
		}
		$in_quotes = 1;
		$have_par = 0;
	} elsif (/^\*\t(.*)$/) {	# Itemized
			if ($debug_pars) {
				&finish_text_close_quote();
				&start_text_open_quote();
			}
		&start_text_open_quote();
		&close_quote();
		if ($in_itemized) {
			print ":Item()\"", &escape_term($1), "\n";
		} else {
			print ":Itemize() (\nItem()\"", &escape_term($1), "\n";
			$in_itemized = 1;
		}
		$in_quotes = 1;
		$have_par = 0;
	}	# Now non-TAB indented markup
	elsif (/^\*INCLUDE\s\s*([^ ].*)\s*$/i) {	# Include another file
		&finish_text_close_quote();
		$have_par = 1;
		print "IncludeFile(\"$1\");\n\n";
	}
	elsif (/^\*EVAL\s\s*([^ ].*)\s*$/i) {	# Evaluate a Yacas expression and insert its result as text
		$expression = $1;
		# un-escape quotes and backslash again
		$expression =~ s/\\"/"/g;
		$expression =~ s/\\\\/\\/g;
		&finish_text_close_quote();
		$have_par = 0;	# even if we had a paragraph before us, now we don't
		print "DocEval($expression);\n";	# this will insert inline Yacas code
	} elsif (/^\*FOOT\s\s*(.*)$/) {	# footnote
		&finish_text_close_quote();
		#$have_par = 0;
		print "AddBody(DocFootnote(\"" . &escape_term($1) . "\"));\n";
	}
	#############################################################
	# stuff for refman
	#############################################################
	elsif (/^\*([-A-Z.]+)\s*$/ and defined($star_labels{$1})) {	# labels without parameters
		&finish_text_close_quote();
		$have_par = 1;
		print $star_labels{$1} . "\n";
	} elsif (/^\*(?:HEAD)\s\s*(.*)$/) {	# Topical()
		&finish_text_close_quote();
		$have_par = 1;
		print "Topical()\"" . &escape_term($1) . "\";\n";
	} elsif (/^\*A\s\s*(.*)$/) {	# anchor
		&finish_text_close_quote();
		#$have_par = 1;
		$anchor = $1;
		# if the first word is {...}, then we need to use the @ stuff
		$anchor = &escape_term($anchor) unless ($anchor =~
			s/^(.*)\{([^{}]+)\}(.*)$/"$1$2$3" . "\@"
			. &escape_term("$1" . "{$2}$3")/e);
		print "AddBody(AddAnchor(\"" . $anchor . "\"));\n";
	} elsif (/^\*SEE\s\s*(.*)$/) {	# SeeAlso()
		$names = $1;
		$names =~ s/\s*$//;
		$names =~ s/^\s*//;
		&finish_text_close_quote();
		$have_par = 1;
		print "Topical()\"See also:\";\nSeeAlso({";
		$have_prev_name = 0;
		foreach $name (split(/,[ \t]+/, $names)) {
			print ((($have_prev_name == 1) ? ", " : "") . "\"$name\"");
			$have_prev_name = 1;
		}
		print "});\n";
	} elsif (/^\*SEE\s*(.*)$/) {	# empty SeeAlso()
		&finish_text_close_quote();
		$have_par = 1;
		# do nothing
	} elsif (/^\*(?:CMD|FUNC)\s\s*(.*)\s*---\s*(.*)$/) {	# CmdDescription()
		$names = $1;
		$title = $2;
		&finish_text_close_quote();
		$have_par = 1;
		print "\n";	# separate from previous command (has no effect on docs)
		# Trim extra spaces
		$names =~ s/\s*$//;
{'fraction_non_alphanumeric': 0.2922551975876845, 'fraction_numerical': 0.010315822885256308, 'mean_word_length': 3.770249810749432, 'pattern_counts': {'":': 12, '<': 5, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6924272', 'n_tokens_mistral': 5264, 'n_tokens_neox': 4759, 'n_words': 1102}