<commit_before>/*
*reverseparser.js
*<NAME>, jemah
*
* Use: call reverseParse(input) where input is the string of song text, returns the markup-language version of the song text. 
*/

/*
Reverse parsing features
Recognize chord line, lyric line, places chords within following lyric line. (chord comments!) 
If chord line above chord line or empty line, print chord line on its own.
If lyric line without preceding chord line, just print lyric line
Recognize common headers like Verse 1, Chorus, Bridge, Intro, etc.
Recognize tablature, enclose in {tab}
Lines directly above Tab also enclosed in tab
Make beginning of document header font, until first chord line seen or section header seen
If repeated section headers, try to write to "order" (always write to order?)

Kinds of Lines: 
chord: common chord forms, more than one space between them, few words, "x#" or "#x" are def. comments
section header: Common name, ends with colon, all caps, ALONE in line
tab: many contiguous '-', with some numbers, few letters (towards beginning/end)
empty: only spaces -- new section
lyric:default, if not others.

*/

var commonheaders = new Array( /^\s*\[?\s*((\d(th|nd|rd|st)?)|last|final)?\s*(verse(s)?([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*((\d(th|nd|rd|st)?)|last|final|pre-?)?\s*(chorus(es)?([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(bridge(s)?([ _]*\d+)?)\s*:?\s*\]?\s*$/i, /^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(intro(duction)?([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(outro([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(instrumental([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(\d* *solo([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(end(ing)?([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(break(down)([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(tag([ _]*\d+)?)\s*:?\s*\]?\s*$/i)
var emptyLine = /^\s*$/;
var hasChords = /(\s|,|-|\(|^)([A-G][#b]?(m|min|dim|maj|sus|aug|\+)?\d{0,2}(sus|add)?\d{0,2}(\/[A-G][#b]?)?)(\s|,|-|\)|$)/g;
var replaceChords = /(\s|,|-|\(|^)?([A-G][#b]?(m|min|dim|maj|sus|aug|\+)?\d{0,2}\(?(sus|add)?\d{0,2}\)?(\/[A-G][#b]?)?)(\s|,|-|\)|$)?/g;
var replaceCommentedChords = /<(\(?[A-G][#b]?(m|min|dim|maj|sus|aug|\+)?\d{0,2}\(?(sus|add)?\d{0,2}\)?(\/[A-G][#b]?)?\)?)>/g;
var justChords = /^((\s+|\s|,|-|\(|)[A-G][#b]?(m|min|dim|maj|sus|aug|\+)?\d{0,2}\(?(sus|add)?\d{0,2}\)?(\/[A-G][#b]?)?(\s+|,|-|\)|))+$/;
var tabLine = /^([A-G][#b]?)*\s*[\|:]*\s*[x\d]*-+[x\d\|-]+/i;

function lineType(line)
{
	//is it empty?
	if(emptyLine.test(line))
		return "empty";
	//is it a header?
	for(var i = 0; i < commonheaders.length ; i+=1)
		if(commonheaders[i].test(line))
			return "header";	
	//is it a tab line?
		if(tabLine.test(line))
			return "tab";
	//is it a chord line?
	if(justChords.test(line))
		return "chord";
	//if more than half of the words are chords
	var words = line.trim().split(/\s+/g)
	var chords = 0;
	for(var i = 0; i < words.length ; i += 1)
		if(justChords.test(words[i])) chords +=1;
	if(chords * 2 >= words.length)
		return "chord";
	//otherwise, a lyric line
	return "lyric";<commit_msg>small reverseparser change<commit_after>/*
*reverseparser.js
*<NAME>, jemah
*
* Use: call reverseParse(input) where input is the string of song text, returns the markup-language version of the song text. 
*/

/*
Reverse parsing features
Recognize chord line, lyric line, places chords within following lyric line. (chord comments!) 
If chord line above chord line or empty line, print chord line on its own.
If lyric line without preceding chord line, just print lyric line
Recognize common headers like Verse 1, Chorus, Bridge, Intro, etc.
Recognize tablature, enclose in {tab}
Lines directly above Tab also enclosed in tab
Make beginning of document header font, until first chord line seen or section header seen
If repeated section headers, try to write to "order" (always write to order?)

Kinds of Lines: 
chord: common chord forms, more than one space between them, few words, "x#" or "#x" are def. comments
section header: Common name, ends with colon, all caps, ALONE in line
tab: many contiguous '-', with some numbers, few letters (towards beginning/end)
empty: only spaces -- new section
lyric:default, if not others.

*/

var commonheaders = new Array( /^\s*\[?\s*((\d(th|nd|rd|st)?)|last|final)?\s*(verse(s)?([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*((\d(th|nd|rd|st)?)|last|final|pre-?)?\s*(chorus(es)?([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(bridge(s)?([ _]*\d+)?)\s*:?\s*\]?\s*$/i, /^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(intro(duction)?([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(outro([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(instrumental([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(\d* *solo([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(end(ing)?([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(break(down)([ _]*\d+)?)\s*:?\s*\]?\s*$/i,/^\s*\[?\s*(\d(th|nd|rd|st)?)?\s*(tag([ _]*\d+)?)\s*:?\s*\]?\s*$/i)
var emptyLine = /^\s*$/;
var hasChords = /(\s|,|-|\(|^)([A-G][#b]?(m|min|dim|maj|sus|aug|\+)?\d{0,2}(sus|add)?\d{0,2}(\/[A-G][#b]?)?)(\s|,|-|\)|$)/g;
var replaceChords = /(\s|,|-|\(|^)?([A-G][#b]?(m|min|dim|maj|sus|aug|\+)?\d{0,2}\(?(sus|add)?\d{0,2}\)?(\/[A-G][#b]?)?)(\s|,|-|\)|$)?/g;
var replaceCommentedChords = /<(\(?[A-G][#b]?(m|min|dim|maj|sus|aug|\+)?\d{0,2}\(?(sus|add)?\d{0,2}\)?(\/[A-G][#b]?)?\)?)>/g;
var justChords = /^((\s+|\s|,|-|\(|)[A-G][#b]?(m|min|dim|maj|sus|aug|\+)?\d{0,2}\(?(sus|add)?\d{0,2}\)?(\/[A-G][#b]?)?(\s+|,|-|\)|))+$/;
var tabLine = /^([A-G][#b]?)*\s*[\|:]*\s*[x\d]*-+[x\d\|-]+/i;

function lineType(line)
{
	//is it empty?
	if(emptyLine.test(line))
		return "empty";
	//is it a header?
	for(var i = 0; i < commonheaders.length ; i+=1)
		if(commonheaders[i].test(line))
			return "header";	
	//is it a tab line?
		if(tabLine.test(line))
			return "tab";
	//is it a chord line?
	if(justChords.test(line))
		return "chord";
	//if more than half of the words are chords
	var words = line.trim().split(/\s+/g)
	var chords = 0;
	for(var i = 0; i < words.length ; i += 1)
		if(justChords.test(words[i])) chords +=1;
	if(chords >= words.length * 3)
		return "chord";
	//otherwise, a lyric line
	return "lyric";
{'fraction_non_alphanumeric': 0.3204175233275344, 'fraction_numerical': 0.00759133322789815, 'mean_word_length': 4.419023136246786, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2609858', 'n_tokens_mistral': 3036, 'n_tokens_neox': 2893, 'n_words': 553}