<commit_before>package persist

import (
	"fmt"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	pclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pfs/db/persist"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfsserver "github.com/pachyderm/pachyderm/src/server/pfs/server"

	"github.com/dancannon/gorethink"
	"go.pedge.io/pb/go/google/protobuf"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"
)

var (
	port int32 = 30651
)

func TestCommitBranchIndexBasicRF(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)

		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.GetAll(commitID).Run(dbClient)
		require.NoError(t, err)
		c := &persist.Commit{}
		err = cursor.One(c)
		require.NoError(t, err)
		require.Equal(t, commitID, c.ID)

		clock := &persist.Clock{Branch: "master", Clock: 0}
		clockID := getClockID(repo.Name, clock).ID
		cursor, err = gorethink.DB(dbName).Table(clockTable).GetAll(clockID).Run(dbClient)
		require.NoError(t, err)
		returnedClock := &persist.Clock{}
		require.NoError(t, cursor.One(returnedClock))
		require.Equal(t, "master", returnedClock.Branch)
		require.Equal(t, uint64(0), returnedClock.Clock)

		key := []interface{}{repo.Name, clock.Branch, clock.Clock}
		cursor, err = term.GetAllByIndex(CommitBranchIndex.GetName(), key).Run(dbClient)
		require.NoError(t, err)
		returnedCommit := &persist.Commit{}
		require.NoError(t, cursor.One(returnedCommit))
		require.Equal(t, commitID, returnedCommit.ID)
	})
}

func TestCommitBranchIndexHeadOfBranchRF(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		commitID2 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID2,
			commitID,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		// master exists, when providing parentID and branch, assume its a new branch
		require.YesError(t, err)

		err = d.StartCommit(
			repo,
			commitID2,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit2 := &pfs.Commit{Repo: repo, ID: commitID2}
		require.NoError(t, d.FinishCommit(commit2, timestampNow(), false, nil))

		commitID3 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID3,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit3 := &pfs.Commit{Repo: repo, ID: commitID3}
		require.NoError(t, d.FinishCommit(commit3, timestampNow(), false, nil))

		// Now that the commits are chained together,
		// Grab the head of master branch using the index
		head := &persist.Commit{}
		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.OrderBy(gorethink.OrderByOpts{
			Index: gorethink.Desc(CommitBranchIndex.GetName()),
		}).Between([]interface{}{repo.Name, "master", 0}, []interface{}{repo.Name, "master", gorethink.MaxVal}, gorethink.BetweenOpts{
			RightBound: "open",
		}).Run(dbClient)
		require.NoError(t, err)
		require.NoError(t, cursor.One(head))
		require.Equal(t, commitID3, head.ID)
	})
}

func TestDiffCommitIndexBasicRF(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
			Path: "file",
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("foo\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		cursor, err := gorethink.DB(dbName).Table(diffTable).Map(DiffPathIndex.GetCreateFunction()).Run(dbClient)
		require.NoError(t, err)
		fields := []interface{}{}
		require.NoError(t, cursor.One(&fields))
		// Example return value:
		// [[foo false file [[master 0]]]]
		innerFields, ok := fields[0].([]interface{})
		require.Equal(t, true, ok)
		require.Equal(t, repo.Name, innerFields[0].(string))
		require.Equal(t, false, innerFields[1].(bool))
		require.Equal(t, "file", innerFields[2].(string))
		clocks, ok := innerFields[3].([]interface{})
		require.Equal(t, true, ok)
		clock, ok := clocks[0].([]interface{})
		require.Equal(t, true, ok)
		require.Equal(t, "master", clock[0].(string))
		require.Equal(t, float64(0), clock[1].(float64))

	})
}

func TestDiffPathIndexBasicRF(t *testing.T) {

	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "repo1"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)<commit_msg>Oops, need to use the proper index in the test<commit_after>package persist

import (
	"fmt"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	pclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	pfsclient "github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"
	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pfs/db/persist"
	"github.com/pachyderm/pachyderm/src/server/pfs/drive"
	pfsserver "github.com/pachyderm/pachyderm/src/server/pfs/server"

	"github.com/dancannon/gorethink"
	"go.pedge.io/pb/go/google/protobuf"
	"go.pedge.io/proto/server"
	"google.golang.org/grpc"
)

var (
	port int32 = 30651
)

func TestCommitBranchIndexBasicRF(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)

		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.GetAll(commitID).Run(dbClient)
		require.NoError(t, err)
		c := &persist.Commit{}
		err = cursor.One(c)
		require.NoError(t, err)
		require.Equal(t, commitID, c.ID)

		clock := &persist.Clock{Branch: "master", Clock: 0}
		clockID := getClockID(repo.Name, clock).ID
		cursor, err = gorethink.DB(dbName).Table(clockTable).GetAll(clockID).Run(dbClient)
		require.NoError(t, err)
		returnedClock := &persist.Clock{}
		require.NoError(t, cursor.One(returnedClock))
		require.Equal(t, "master", returnedClock.Branch)
		require.Equal(t, uint64(0), returnedClock.Clock)

		key := []interface{}{repo.Name, clock.Branch, clock.Clock}
		cursor, err = term.GetAllByIndex(CommitBranchIndex.GetName(), key).Run(dbClient)
		require.NoError(t, err)
		returnedCommit := &persist.Commit{}
		require.NoError(t, cursor.One(returnedCommit))
		require.Equal(t, commitID, returnedCommit.ID)
	})
}

func TestCommitBranchIndexHeadOfBranchRF(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		commitID2 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID2,
			commitID,
			"master",
			timestampNow(),
			nil,
			nil,
		)
		// master exists, when providing parentID and branch, assume its a new branch
		require.YesError(t, err)

		err = d.StartCommit(
			repo,
			commitID2,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit2 := &pfs.Commit{Repo: repo, ID: commitID2}
		require.NoError(t, d.FinishCommit(commit2, timestampNow(), false, nil))

		commitID3 := uuid.NewWithoutDashes()
		err = d.StartCommit(
			repo,
			commitID3,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		commit3 := &pfs.Commit{Repo: repo, ID: commitID3}
		require.NoError(t, d.FinishCommit(commit3, timestampNow(), false, nil))

		// Now that the commits are chained together,
		// Grab the head of master branch using the index
		head := &persist.Commit{}
		term := gorethink.DB(dbName).Table(commitTable)
		cursor, err := term.OrderBy(gorethink.OrderByOpts{
			Index: gorethink.Desc(CommitBranchIndex.GetName()),
		}).Between([]interface{}{repo.Name, "master", 0}, []interface{}{repo.Name, "master", gorethink.MaxVal}, gorethink.BetweenOpts{
			RightBound: "open",
		}).Run(dbClient)
		require.NoError(t, err)
		require.NoError(t, cursor.One(head))
		require.Equal(t, commitID3, head.ID)
	})
}

func TestDiffCommitIndexBasicRF(t *testing.T) {
	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "foo"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
		file := &pfs.File{
			Commit: &pfs.Commit{
				Repo: repo,
				ID:   commitID,
			},
			Path: "file",
		}
		d.PutFile(file, "", pfs.Delimiter_LINE, 0, strings.NewReader("foo\n"))

		commit := &pfs.Commit{Repo: repo, ID: commitID}
		require.NoError(t, d.FinishCommit(commit, timestampNow(), false, nil))

		cursor, err := gorethink.DB(dbName).Table(diffTable).Map(DiffCommitIndex.GetCreateFunction()).Run(dbClient)
		require.NoError(t, err)
		var indexedAsCommit string
		require.NoError(t, cursor.One(&indexedAsCommit))
		require.Equal(t, commitID, indexedAsCommit)
	})
}

func TestDiffPathIndexBasicRF(t *testing.T) {

	testSetup(t, func(d drive.Driver, dbName string, dbClient *gorethink.Session, client pclient.APIClient) {

		repo := &pfs.Repo{Name: "repo1"}
		require.NoError(t, d.CreateRepo(repo, timestampNow(), nil, nil))
		commitID := uuid.NewWithoutDashes()
		err := d.StartCommit(
			repo,
			commitID,
			"",
			"master",
			timestampNow(),
			nil,
			nil,
		)
		require.NoError(t, err)
{'fraction_non_alphanumeric': 0.16064152579107066, 'fraction_numerical': 0.005721716514954486, 'mean_word_length': 4.296602387511478, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4205298', 'n_tokens_mistral': 4821, 'n_tokens_neox': 4436, 'n_words': 781}