<commit_before>package main

import (
	"io/ioutil"
	"os"
	"testing"
	"time"

	"labix.org/v2/mgo"

	"github.com/stretchr/testify/assert"

	bsonScanner "github.com/Clever/oplog-replay/bson"
)

// simpleDocStruct is for inserting simple Mongo documents
type simpleDocStruct struct {
	key string
}

func TestComposingWithOplogReplay(t *testing.T) {
	// This test assumes that we're connecting to Mongo replica set. Otherwise, the oplog won't
	// be generated.

	unixTime := int(time.Now().Unix())

	time.Sleep(time.Duration(2) * time.Second)

	session, err := mgo.Dial("localhost")
	assert.Nil(t, err)
	db := session.DB("myTestDb")
	c := db.C("myCollection")
	assert.Nil(t, c.Insert(&simpleDocStruct{key: "key"}))

	time.Sleep(time.Duration(2) * time.Second)

	assert.Nil(t, c.Insert(&simpleDocStruct{key: "key2"}))
	// Dump at the time we started operations. Should get both operations
	dumpAtTime(t, unixTime, 2)
	// Three seconds later we should get only one.
	dumpAtTime(t, unixTime+3, 1)
}

func dumpAtTime(t *testing.T, unixTime, expectedResults int) {
	tempDir, err := ioutil.TempDir("/tmp", "oplogDumpTest")
	assert.Nil(t, err)
	defer os.RemoveAll(tempDir)
	assert.Nil(t, runDump(tempDir, "localhost", "", unixTime))

	file, err := os.Open(tempDir + "/local/oplog.rs.bson")
	assert.Nil(t, err)
	scanner := bsonScanner.New(file)
	count := 0
	for scanner.Scan() {
		count = count + 1
	}
	assert.Equal(t, expectedResults, count)<commit_msg>Add test for filtration by collection<commit_after>package main

import (
	"io/ioutil"
	"os"
	"testing"
	"time"

	"labix.org/v2/mgo"

	"github.com/stretchr/testify/assert"

	bsonScanner "github.com/Clever/oplog-replay/bson"
)

// simpleDocStruct is for inserting simple Mongo documents
type simpleDocStruct struct {
	key string
}

func getPaddedTime() (n int) {
	time.Sleep(time.Duration(1) * time.Second)
	n = int(time.Now().Unix())
	time.Sleep(time.Duration(2) * time.Second)
	return
}

func TestComposingWithOplogReplay(t *testing.T) {
	// This test assumes that we're connecting to Mongo replica set. Otherwise, the oplog won't
	// be generated.

	unixTime := getPaddedTime()
	session, err := mgo.Dial("localhost")
	assert.Nil(t, err)
	db := session.DB("myTestDb")
	c := db.C("myCollection")
	assert.Nil(t, c.Insert(&simpleDocStruct{key: "key"}))

	time.Sleep(time.Duration(2) * time.Second)

	assert.Nil(t, c.Insert(&simpleDocStruct{key: "key2"}))
	// Dump at the time we started operations. Should get both operations
	dumpAtTime(t, unixTime, 2, "")
	// Three seconds later we should get only one.
	dumpAtTime(t, unixTime+3, 1, "")
}

func TestCollectionFiltering(t *testing.T) {
	// This test assumes that we're connecting to Mongo replica set. Otherwise, the oplog won't
	// be generated.

	unixTime := getPaddedTime()
	session, err := mgo.Dial("localhost")
	assert.Nil(t, err)
	db := session.DB("myTestDb")
	c1, c2, c3 := db.C("myCollection"), db.C("yourCollection"), db.C("ourCollection")

	assert.Nil(t, c1.Insert(&simpleDocStruct{key: "key"}))
	assert.Nil(t, c2.Insert(&simpleDocStruct{key: "key2"}))
	assert.Nil(t, c3.Insert(&simpleDocStruct{key: "key3"}))

	dumpAtTime(t, unixTime, 3, "")
	dumpAtTime(t, unixTime, 1, "myTestDb.myCollection")
	dumpAtTime(t, unixTime, 1, "myTestDb.yourCollection")
	dumpAtTime(t, unixTime, 2, "{$ne : \"myTestDb.myCollection\"}")
}

func dumpAtTime(t *testing.T, unixTime, expectedResults int, collection string) {
	tempDir, err := ioutil.TempDir("/tmp", "oplogDumpTest")
	assert.Nil(t, err)
	defer os.RemoveAll(tempDir)
	assert.Nil(t, runDump(tempDir, "localhost", collection, unixTime))

	file, err := os.Open(tempDir + "/local/oplog.rs.bson")
	assert.Nil(t, err)
	scanner := bsonScanner.New(file)
	count := 0
	for scanner.Scan() {
		count = count + 1
	}
	assert.Equal(t, expectedResults, count)
{'fraction_non_alphanumeric': 0.15776955602537, 'fraction_numerical': 0.008192389006342496, 'mean_word_length': 4.76103500761035, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1762331', 'n_tokens_mistral': 1478, 'n_tokens_neox': 1419, 'n_words': 323}