<commit_before>package main

import (
	"bytes"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Filter", func() {

	Describe("Smudge filter", func() {

		It("doesn't alter non-LOB content", func() {
			nonLOBString := `This is some data
in a string
that we should absolutely not mess with`
			inBuffer := bytes.NewBufferString(nonLOBString)
			var outBuffer bytes.Buffer
			res := SmudgeFilterWithReaderWriter(inBuffer, &outBuffer)
			Expect(res).To(Equal(0), "smudge filter should succeed")
			Expect(outBuffer.String()).To(BeEquivalentTo(nonLOBString), "non LOB should not be modified by smudge")
		})

		It("doesn't alter LOB content when LOB isn't present in object store & no autodownloading", func() {
			// TODO this is when auto download is not implemented; turn it off when added
			// Made up SHA that doesn't exist
			lobString := SHAPrefix + "0123456789abcdef0123456789abcdef01234567"
			inBuffer := bytes.NewBufferString(lobString)
			var outBuffer bytes.Buffer
			res := SmudgeFilterWithReaderWriter(inBuffer, &outBuffer)
			Expect(res).To(Equal(0), "smudge filter should succeed")
			Expect(outBuffer.String()).To(BeEquivalentTo(lobString), "non existent LOB should not be modified by smudge")
		})

	})

})<commit_msg>Completed tests for smudge filter<commit_after>package main

import (
	"bytes"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Filter", func() {

	Describe("Smudge filter", func() {

		It("doesn't alter non-LOB content", func() {
			nonLOBString := `This is some data
in a string
that we should absolutely not mess with`
			inBuffer := bytes.NewBufferString(nonLOBString)
			var outBuffer bytes.Buffer
			res := SmudgeFilterWithReaderWriter(inBuffer, &outBuffer)
			Expect(res).To(Equal(0), "smudge filter should succeed")
			Expect(outBuffer.String()).To(BeEquivalentTo(nonLOBString), "non LOB should not be modified by smudge")
		})

		It("doesn't alter LOB content when LOB isn't present in object store & no autodownloading", func() {
			// TODO this is when auto download is not implemented; turn it off when added
			// Made up SHA that doesn't exist
			lobString := SHAPrefix + "0123456789abcdef0123456789abcdef01234567"
			inBuffer := bytes.NewBufferString(lobString)
			var outBuffer bytes.Buffer
			res := SmudgeFilterWithReaderWriter(inBuffer, &outBuffer)
			Expect(res).To(Equal(0), "smudge filter should succeed")
			Expect(outBuffer.String()).To(BeEquivalentTo(lobString), "non existent LOB should not be modified by smudge")
		})

		It("writes real LOB data for small file", func() {
			lobinfo := CreateSmallTestLOBDataForRetrieval()
			lobString := SHAPrefix + lobinfo.SHA
			inBuffer := bytes.NewBufferString(lobString)
			var outBuffer bytes.Buffer
			res := SmudgeFilterWithReaderWriter(inBuffer, &outBuffer)
			Expect(res).To(Equal(0), "smudge filter should succeed")
			Expect(outBuffer.Len()).To(BeEquivalentTo(lobinfo.Size), "extracted LOB data should be correct size")
		})

		It("writes real LOB data for large file [LONGTEST]", func() {
			lobinfo := CreateLargeTestLOBDataForRetrieval()
			lobString := SHAPrefix + lobinfo.SHA
			inBuffer := bytes.NewBufferString(lobString)
			var outBuffer bytes.Buffer
			res := SmudgeFilterWithReaderWriter(inBuffer, &outBuffer)
			Expect(res).To(Equal(0), "smudge filter should succeed")
			Expect(outBuffer.Len()).To(BeEquivalentTo(lobinfo.Size), "extracted LOB data should be correct size")
		})

	})

})
{'fraction_non_alphanumeric': 0.11550678602367889, 'fraction_numerical': 0.017903551833670227, 'mean_word_length': 4.792642140468227, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2587490', 'n_tokens_mistral': 1235, 'n_tokens_neox': 1096, 'n_words': 301}