<commit_before>package bbConvert

import "testing"

func TestStandard(t *testing.T) {
	testStr := "[ol][b]Item 1[/b]\n[i]Item 2[/i]\n[font='Times New Roman' size=20 color=red variant=smallcaps]Item 3[/font][/ol]\nNext Paragraph\nAnd the next\n[i]<Orphan tag :)"
	conv := CreateConverter(true, true)
	t.Log(conv.Convert(testStr))
}

func TestNoWraping(t *testing.T) {
	testStr := "[u][i]Hello[s]UnderItalStri[/s][/i][/u][b]<another orphan tag :)"
	conv := CreateConverter(false, true)
	t.Log(conv.Convert(testStr))
}

func TestMarkdown(t *testing.T) {
	testStr := "[bold]Simple Bold[/bold] [italics]Simple Italics[/italics]"
	conv := CreateConverter(false, false)
	conv.AddCustomTag("bold", func(fnt Tag, meat string) string {
		return "**" + meat + "**"
	})
	conv.AddCustomTag("italics", func(fnt Tag, meat string) string {
		return "*" + meat + "*"
	})
	t.Log(conv.Convert(testStr))
}<commit_msg>Adding more coverage. Has trouble with the ultimateString<commit_after>package bbConvert

import "testing"

func TestStandard(t *testing.T) {
	testStr := "[ol][b]Item 1[/b]\n[i]Item 2[/i]\n[font='Times New Roman' size=20 color=red variant=smallcaps]Item 3[/font][/ol]\nNext Paragraph\nAnd the next\n[i]<Orphan tag :)"
	conv := CreateConverter(true, true)
	t.Log(conv.Convert(testStr))
}

func TestNoWraping(t *testing.T) {
	testStr := "[u][i]Hello[s]UnderItalStri[/s][/i][/u][b]<another orphan tag :)"
	conv := CreateConverter(false, true)
	t.Log(conv.Convert(testStr))
}

func TestMarkdown(t *testing.T) {
	testStr := "[bold]Simple Bold[/bold] [italics]Simple Italics[/italics]"
	conv := CreateConverter(false, false)
	conv.AddCustomTag("bold", func(fnt Tag, meat string) string {
		return "**" + meat + "**"
	})
	conv.AddCustomTag("italics", func(fnt Tag, meat string) string {
		return "*" + meat + "*"
	})
	t.Log(conv.Convert(testStr))
}

func TestAllTags(t *testing.T) {
	ultimateStr := "[b]Bold[/b][bold]Bold[/bold][i]Italics[/i][italics]Italics[/italics][u]Underline[/u][underline]Underline[/underline][s]Strike[/s][strike]Strike[/strike][font=Verdana]Some Text[/font][font size=20pt]Some Text[/font][font color=red]Some Text[/font][font color=#000000]Some Text[/font][font variant=upper]Some Text[/font][font variant=lower]Some Text[/font][font variant=smallcaps]Some Text[/font][size=20pt]Some Text[/size][color=red]Some Text[/color][color=#000000]Some Text[/color][smallcaps]Some Text[/smallcaps][url]Link address[/url][url=address]Some Text[/url][url title='Title']Link address[/url][link]Link address[/link][link=address]Some Text[/link][link title='Title']Link address[/link][youtube]Youtube URL or video ID[/youtube][youtube height=200 width=500]Youtube URL or video ID[/youtube][youtube=500x200]Youtube URL or video ID[/youtube][youtube left]Youtube URL or video ID[/youtube][youtube right]Youtube URL or video ID[/youtube][img]Image URL[/img][img=500x200]Image URL[/img][img height=200 width=500]Image URL[/img][img left]Image URL[/img][img right]Image URL[/img][img alt='Alternate text']Image URL[/img][img title='Title']Image URL[/img][image]Image URL[/image][title]Some Text[/title][t1]Some Text[/t1][t2]Some Text[/t2][t3]Some Text[/t3][t4]Some Text[/t4][t5]Some Text[/t5][t6]Some Text[/t6][align=center]Some Text[/align][ul]\n* Item 1\nItem 2\n[/ul][ol]\n* Item 1\nItem 2\n[/ol][bullet] * Item 1 * Item 2[/bullet][number] * Item 1 * Item 2[/number][ul]* Item 1 * Item 2[/ul][ol]* Item 1 * Item 2[/ol]"
	conv := CreateConverter(true, true)
	t.Log(conv.Convert(ultimateStr))
}
{'fraction_non_alphanumeric': 0.19450014322543685, 'fraction_numerical': 0.02119736465196219, 'mean_word_length': 6.641137855579869, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1945870', 'n_tokens_mistral': 1377, 'n_tokens_neox': 1234, 'n_words': 238}