<commit_before>// contains the tests for the metadata tests

package gnosis

import (
	"github.com/stretchr/testify/assert"
	//"os"
	//"path"
	"testing"
)

func stringKeyExistsInMap(itemMap map[string]bool, key string) bool {
	defer func() bool {
		r := recover()
		return r == nil
	}()
	junk := itemMap[key]
	return junk == junk
}

func TestLineIsTitle(t *testing.T) {
	pdata := new(PageMetadata)

	// test the most normal topic line I'd expect
	titleLine := []byte("=======")
	assert.True(t, pdata.lineIsTitle(titleLine), "the most basic topic line just failed")

	titleLine = []byte("=")
	assert.True(t, pdata.lineIsTitle(titleLine), "one = should be enough")

	titleLine = []byte("   ======")
	assert.True(t, pdata.lineIsTitle(titleLine), "any spaces before the heading portion should not cause failure")

	titleLine = []byte("			======")
	assert.True(t, pdata.lineIsTitle(titleLine), "tabs before the heading portion should not cause failure")

	titleLine = []byte("=======     ")
	assert.True(t, pdata.lineIsTitle(titleLine), "spaces after the heading portion should not cause failure")

	titleLine = []byte("=======			")
	assert.True(t, pdata.lineIsTitle(titleLine), "tabs after the heading portion should not cause failure")

	titleLine = []byte("=======\n")
	assert.True(t, pdata.lineIsTitle(titleLine), "a newline after the heading portion should not cause failure")

	titleLine = []byte("===== ===")
	assert.False(t, pdata.lineIsTitle(titleLine), "the underlining has to be continous - no spaces - so this should have failed")

	titleLine = []byte("====	=====")
	assert.False(t, pdata.lineIsTitle(titleLine), "the underlining has to be continous - no tabs - so this should have failed")
}

func TestProcessMetadata(t *testing.T) {
	pdata := new(PageMetadata)

	metaDataLine := []byte("topic=a")
	pdata.processMetadata(metaDataLine)
	assert.True(t, stringKeyExistsInMap(pdata.Topics, "a"), "the element I just tried to add should have been added")
	assert.True(t, pdata.Topics["a"], , "the element I just tried to add should have been added")
}<commit_msg>got all checks within TestCheckMatch working<commit_after>// contains the tests for the metadata tests

package gnosis

import (
	"github.com/stretchr/testify/assert"
	//"os"
	//"path"
	"testing"
)

func stringKeyExistsInMap(itemMap map[string]bool, key string) bool {
	defer func() bool {
		r := recover()
		return r == nil
	}()
	junk := itemMap[key]
	return junk == junk
}

func TestLineIsTitle(t *testing.T) {
	pdata := new(PageMetadata)

	// test the most normal topic line I'd expect
	titleLine := []byte("=======")
	assert.True(t, pdata.lineIsTitle(titleLine), "the most basic topic line just failed")

	titleLine = []byte("=")
	assert.True(t, pdata.lineIsTitle(titleLine), "one = should be enough")

	titleLine = []byte("   ======")
	assert.True(t, pdata.lineIsTitle(titleLine), "any spaces before the heading portion should not cause failure")

	titleLine = []byte("			======")
	assert.True(t, pdata.lineIsTitle(titleLine), "tabs before the heading portion should not cause failure")

	titleLine = []byte("=======     ")
	assert.True(t, pdata.lineIsTitle(titleLine), "spaces after the heading portion should not cause failure")

	titleLine = []byte("=======			")
	assert.True(t, pdata.lineIsTitle(titleLine), "tabs after the heading portion should not cause failure")

	titleLine = []byte("=======\n")
	assert.True(t, pdata.lineIsTitle(titleLine), "a newline after the heading portion should not cause failure")

	titleLine = []byte("===== ===")
	assert.False(t, pdata.lineIsTitle(titleLine), "the underlining has to be continous - no spaces - so this should have failed")

	titleLine = []byte("====	=====")
	assert.False(t, pdata.lineIsTitle(titleLine), "the underlining has to be continous - no tabs - so this should have failed")
}

func TestCheckMatch(t *testing.T) {
	pdata := new(PageMetadata)

	metadataLine := []byte("topic = a")
	metadataMatch := []byte("topic")
	pdata.checkMatch(metadataLine, metadataMatch, &pdata.Topics)
	assert.True(t, pdata.Topics["a"], "should have been able to add element a to the thingy")

	metadataLine = []byte("topic= b")
	metadataMatch = []byte("topic")
	pdata.checkMatch(metadataLine, metadataMatch, &pdata.Topics)
	assert.True(t, pdata.Topics["b"], "should have been able to add element b to the thingy")

	metadataLine = []byte("topic=c")
	metadataMatch = []byte("topic")
	pdata.checkMatch(metadataLine, metadataMatch, &pdata.Topics)
	assert.True(t, pdata.Topics["c"], "should have been able to add element c to the thingy")

	metadataLine = []byte("topic=d e f")
	metadataMatch = []byte("topic")
	pdata.checkMatch(metadataLine, metadataMatch, &pdata.Topics)
	assert.True(t, pdata.Topics["d-e-f"], "should have been able to add element d e f to the thingy")

	metadataLine = []byte("topic=g  h")
	metadataMatch = []byte("topic")
	pdata.checkMatch(metadataLine, metadataMatch, &pdata.Topics)
	assert.True(t, pdata.Topics["g-h"], "should have been able to add element g-h to the thingy")

	metadataLine = []byte("topic:i")
	metadataMatch = []byte("topic")
	pdata.checkMatch(metadataLine, metadataMatch, &pdata.Topics)
	assert.True(t, pdata.Topics["i"], "should have been able to add element i to the thingy")
}

/*
func TestProcessMetadata(t *testing.T) {
	pdata := new(PageMetadata)

	metaDataLine := []byte("topic = a")
	pdata.processMetadata(metaDataLine)
	assert.True(t, stringKeyExistsInMap(pdata.Topics, "a"), "the element I just tried to add should have been added")
	assert.True(t, pdata.Topics["a"], "the element I just tried to add should be true")
	assert.False(t, pdata.Topics["b"], "this is a trap")

	/*
		metaDataLine = []byte("topic=a")
		pdata.processMetadata(metaDataLine)
		assert.True(t, stringKeyExistsInMap(pdata.Topics, "a"), "the element I just tried to add should have been added")
		assert.True(t, pdata.Topics["a"], "the element I just tried to add should have been added")
*/
//}
{'fraction_non_alphanumeric': 0.1568726781492739, 'fraction_numerical': 0.0, 'mean_word_length': 5.125129265770424, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3464409', 'n_tokens_mistral': 1958, 'n_tokens_neox': 1916, 'n_words': 577}