<commit_before>// contains the tests for the metadata tests

package gnosis

import (
	"github.com/stretchr/testify/assert"
	//"os"
	//"path"
	"testing"
)

func TestLineIsTitle(t *testing.T) {
	pd := new(PageMetadata)

	// test the most normal topic line I'd expect
	titleLine := []byte("=======")
	assert.True(t, pd.lineIsTitle(titleLine), "the most basic topic line just failed")

	titleLine = []byte("=")
	assert.True(t, pd.lineIsTitle(titleLine), "one = should be enough")

	titleLine = []byte("   ======")
	assert.True(t, pd.lineIsTitle(titleLine), "any spaces before the heading portion should not cause failure")

	titleLine = []byte("			======")
	assert.True(t, pd.lineIsTitle(titleLine), "tabs before the heading portion should not cause failure")

	titleLine = []byte("=======     ")
	assert.True(t, pd.lineIsTitle(titleLine), "spaces after the heading portion should not cause failure")

	titleLine = []byte("=======			")
	assert.True(t, pd.lineIsTitle(titleLine), "tabs after the heading portion should not cause failure")

	titleLine = []byte("=======\n")
	assert.True(t, pd.lineIsTitle(titleLine), "a newline after the heading portion should not cause failure")

	titleLine = []byte("===== ===")
	assert.False(t, pd.lineIsTitle(titleLine), "the underlining has to be continous - no spaces - so this should have failed")

	titleLine = []byte("====	=====")
	assert.False(t, pd.lineIsTitle(titleLine), "the underlining has to be continous - no tabs - so this should have failed")
}
<commit_msg>added a test existsInMap to check to see if a given string key exists within a map[string]bool<commit_after>// contains the tests for the metadata tests

package gnosis

import (
	"github.com/stretchr/testify/assert"
	//"os"
	//"path"
	"testing"
)

func existsInMap(itemMap map[string]bool, key string) bool {
	defer func() bool {
		r := recover()
		return r == nil
	}()
	junk := itemMap[key]
	return junk == junk
}

func TestLineIsTitle(t *testing.T) {
	pdata := new(PageMetadata)

	// test the most normal topic line I'd expect
	titleLine := []byte("=======")
	assert.True(t, pd.lineIsTitle(titleLine), "the most basic topic line just failed")

	titleLine = []byte("=")
	assert.True(t, pd.lineIsTitle(titleLine), "one = should be enough")

	titleLine = []byte("   ======")
	assert.True(t, pd.lineIsTitle(titleLine), "any spaces before the heading portion should not cause failure")

	titleLine = []byte("			======")
	assert.True(t, pd.lineIsTitle(titleLine), "tabs before the heading portion should not cause failure")

	titleLine = []byte("=======     ")
	assert.True(t, pd.lineIsTitle(titleLine), "spaces after the heading portion should not cause failure")

	titleLine = []byte("=======			")
	assert.True(t, pd.lineIsTitle(titleLine), "tabs after the heading portion should not cause failure")

	titleLine = []byte("=======\n")
	assert.True(t, pd.lineIsTitle(titleLine), "a newline after the heading portion should not cause failure")

	titleLine = []byte("===== ===")
	assert.False(t, pd.lineIsTitle(titleLine), "the underlining has to be continous - no spaces - so this should have failed")

	titleLine = []byte("====	=====")
	assert.False(t, pd.lineIsTitle(titleLine), "the underlining has to be continous - no tabs - so this should have failed")
}

{'fraction_non_alphanumeric': 0.1640098099325567, 'fraction_numerical': 0.0, 'mean_word_length': 4.7752212389380535, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4499861', 'n_tokens_mistral': 1068, 'n_tokens_neox': 1020, 'n_words': 331}