<commit_before>package main

import (
	"errors"
	"fmt"
	. "github.com/smartystreets/goconvey/convey"
	"os"
	"strings"
	"testing"
	"time"
)

type testWriteCloser struct {
	failOnWrite  bool
	bytesWritten int
	closed       bool
}

func (twc *testWriteCloser) Write(value []byte) (int, error) {
	if twc.failOnWrite {
		return 0, errors.New("Some error")
	}
	twc.bytesWritten = twc.bytesWritten + len(value)
	return len(value), nil
}

func (twc *testWriteCloser) Close() error {
	if twc.closed {
		return errors.New("double close")
	}
	twc.closed = true
	return nil
}

type testFDBFileInfo struct {
	rev int
}

func (tInfo *testFDBFileInfo) Name() string {
	return fmt.Sprintf("%08d", tInfo.rev)
}
func (tInfo *testFDBFileInfo) Size() int64 { return 0 }
func (tInfo *testFDBFileInfo) Mode() os.FileMode {
	return os.FileMode(0750)
}
func (tInfo *testFDBFileInfo) ModTime() time.Time {
	return time.Now()
}
func (tInfo *testFDBFileInfo) IsDir() bool      { return false }
func (tInfo *testFDBFileInfo) Sys() interface{} { return nil }

func TestWikiWordRe(t *testing.T) {
	Convey("All WikiWords match a regexp", t, func() {
		So(IsWikiWord("abcd"), ShouldBeFalse)
		So(IsWikiWord("123Abc"), ShouldBeFalse)
		So(IsWikiWord("abcDef"), ShouldBeFalse)
		So(IsWikiWord("Abc+d3"), ShouldBeFalse)
		So(IsWikiWord("Abc"), ShouldBeFalse)
		So(IsWikiWord("Jonathan@Hanks"), ShouldBeFalse)
		So(IsWikiWord("FirstLast"), ShouldBeTrue)
		So(IsWikiWord("FirstLast123"), ShouldBeTrue)
		So(IsWikiWord("FirstLastOther"), ShouldBeTrue)
	})
}

func TestExtractWikiWords(t *testing.T) {
	test0 := []byte("There are no wiki words in\nthis piece of text.")
	test1 := []byte("There is only\nOneWikiWord in this text.")
	test4 := []byte("OneWikiWord not a wiki word\nTwoWikiWord ThreeWikiWords\n FourWords")
	testMulti := []byte("WordWordOne\nWordWord\n WordOne\nWordWordOne WordWordOne")

	Convey("A string/[]byte may have multiple wikiwords in it", t, func() {
		words0 := ExtractWikiWords(test0)
		So(len(words0), ShouldEqual, 0)

		words1 := ExtractWikiWords(test1)
		So(len(words1), ShouldEqual, 1)

		words4 := ExtractWikiWords(test4)
		So(len(words4), ShouldEqual, 4)

		wordsMulti := ExtractWikiWords(testMulti)
		So(len(wordsMulti), ShouldEqual, 3)
	})
}
<commit_msg>Additional WikiWord tests<commit_after>package main

import (
	"errors"
	"fmt"
	. "github.com/smartystreets/goconvey/convey"
	"os"
	"strings"
	"testing"
	"time"
)

type testWriteCloser struct {
	failOnWrite  bool
	bytesWritten int
	closed       bool
}

func (twc *testWriteCloser) Write(value []byte) (int, error) {
	if twc.failOnWrite {
		return 0, errors.New("Some error")
	}
	twc.bytesWritten = twc.bytesWritten + len(value)
	return len(value), nil
}

func (twc *testWriteCloser) Close() error {
	if twc.closed {
		return errors.New("double close")
	}
	twc.closed = true
	return nil
}

type testFDBFileInfo struct {
	rev int
}

func (tInfo *testFDBFileInfo) Name() string {
	return fmt.Sprintf("%08d", tInfo.rev)
}
func (tInfo *testFDBFileInfo) Size() int64 { return 0 }
func (tInfo *testFDBFileInfo) Mode() os.FileMode {
	return os.FileMode(0750)
}
func (tInfo *testFDBFileInfo) ModTime() time.Time {
	return time.Now()
}
func (tInfo *testFDBFileInfo) IsDir() bool      { return false }
func (tInfo *testFDBFileInfo) Sys() interface{} { return nil }

func TestWikiWordRe(t *testing.T) {
	Convey("All WikiWords match a regexp", t, func() {
		So(IsWikiWord("abcd"), ShouldBeFalse)
		So(IsWikiWord("123"), ShouldBeFalse)
		So(IsWikiWord("123Abc"), ShouldBeFalse)
		So(IsWikiWord("abcDef"), ShouldBeFalse)
		So(IsWikiWord("Abc+d3"), ShouldBeFalse)
		So(IsWikiWord("Abc_Def"), ShouldBeTrue)
		So(IsWikiWord("AbcD_ef"), ShouldBeTrue)
		So(IsWikiWord("AbcD_123"), ShouldBeTrue)
		So(IsWikiWord("Abc"), ShouldBeFalse)
		So(IsWikiWord("Jonathan@Hanks"), ShouldBeFalse)
		So(IsWikiWord("FirstLast"), ShouldBeTrue)
		So(IsWikiWord("FirstLast123"), ShouldBeTrue)
		So(IsWikiWord("FirstLastOther"), ShouldBeTrue)
	})
}

func TestExtractWikiWords(t *testing.T) {
	test0 := []byte("There are no wiki words in\nthis piece of text.")
	test1 := []byte("There is only\nOneWikiWord in this text.")
	test4 := []byte("OneWikiWord not a wiki word\nTwoWikiWord ThreeWikiWords\n FourWords")
	testMulti := []byte("WordWordOne\nWordWord\n WordOne\nWordWordOne WordWordOne")

	Convey("A string/[]byte may have multiple wikiwords in it", t, func() {
		words0 := ExtractWikiWords(test0)
		So(len(words0), ShouldEqual, 0)

		words1 := ExtractWikiWords(test1)
		So(len(words1), ShouldEqual, 1)

		words4 := ExtractWikiWords(test4)
		So(len(words4), ShouldEqual, 4)

		wordsMulti := ExtractWikiWords(testMulti)
		So(len(wordsMulti), ShouldEqual, 3)
	})
}

{'fraction_non_alphanumeric': 0.14120667522464697, 'fraction_numerical': 0.01540436456996149, 'mean_word_length': 5.360544217687075, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6364284', 'n_tokens_mistral': 1879, 'n_tokens_neox': 1752, 'n_words': 390}