<commit_before>package git

import (
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/inconshreveable/log15"
	"go.polydawn.net/meep"

	"go.polydawn.net/repeatr/api/def"
	"go.polydawn.net/repeatr/lib/fs"
	"go.polydawn.net/repeatr/rio"
	"go.polydawn.net/repeatr/rio/transmat/mixins"
)

const Kind = rio.TransmatKind("git")

var _ rio.Transmat = &GitTransmat{}

type GitTransmat struct {
	workArea workArea
}

var _ rio.TransmatFactory = New

func New(workPath string) rio.Transmat {
	mustDir(workPath)
	workPath, err := filepath.Abs(workPath)
	if err != nil {
		panic(meep.Meep(
			&rio.ErrInternal{Msg: "Unable to set up workspace"},
			meep.Cause(err),
		))
	}
	wa := workArea{
		fullCheckouts:  filepath.Join(workPath, "full"),
		nosubCheckouts: filepath.Join(workPath, "nosub"),
		gitDirs:        filepath.Join(workPath, "gits"),
	}
	mustDir(wa.fullCheckouts)
	mustDir(wa.nosubCheckouts)
	mustDir(wa.gitDirs)
	return &GitTransmat{wa}
}

/*
	Git transmats plonk down the contents of one commit (or tree) as a filesystem.

	A fileset materialized by git does *not* include the `.git` dir by default,
	since those files are not themselves part of what's described by the hash.

	Git effectively "filters" out several attributes -- permissions are only loosely
	respected (execution only), file timestamps are undefined, uid/gid bits
	are not tracked, xattrs are not tracked, etc.  If you desired defined values,
	*you must still configure materialization to use a filter* (particularly for
	file timestamps, since they will otherwise be allowed to vary from one
	materialization to the next(!)).

	Git also allows for several other potential pitfalls with lossless data
	transmission: git cannot transmit empty directories.  This can be a major pain.
	Typical workarounds include creating a ".gitkeep" file in the empty directory.
	Gitignore files may also inadventantly cause trouble.  Transmat.Materialize
	will act *consistently*, but it does not overcome these issues in git
	(doing so would require additional metadata or protocol extensions).

	This transmat is *not* currently well optimized, and should generally be assumed
	to be re-cloning on all materializations -- specifically, it is not smart
	enough to recognize requests for different commits and trees from the
	same repos in order to save reclones.
*/
func (t *GitTransmat) Materialize(
	kind rio.TransmatKind,
	dataHash rio.CommitID,
	siloURIs []rio.SiloURI,
	log log15.Logger,
	options ...rio.MaterializerConfigurer,
) rio.Arena {
	var arena gitArena
	meep.Try(func() {
		// Basic validation and config
		mixins.MustBeType(Kind, kind)<commit_msg>Remove admonishment in git transmat docs; it has in fact improved well beyond that point!<commit_after>package git

import (
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/inconshreveable/log15"
	"go.polydawn.net/meep"

	"go.polydawn.net/repeatr/api/def"
	"go.polydawn.net/repeatr/lib/fs"
	"go.polydawn.net/repeatr/rio"
	"go.polydawn.net/repeatr/rio/transmat/mixins"
)

const Kind = rio.TransmatKind("git")

var _ rio.Transmat = &GitTransmat{}

type GitTransmat struct {
	workArea workArea
}

var _ rio.TransmatFactory = New

func New(workPath string) rio.Transmat {
	mustDir(workPath)
	workPath, err := filepath.Abs(workPath)
	if err != nil {
		panic(meep.Meep(
			&rio.ErrInternal{Msg: "Unable to set up workspace"},
			meep.Cause(err),
		))
	}
	wa := workArea{
		fullCheckouts:  filepath.Join(workPath, "full"),
		nosubCheckouts: filepath.Join(workPath, "nosub"),
		gitDirs:        filepath.Join(workPath, "gits"),
	}
	mustDir(wa.fullCheckouts)
	mustDir(wa.nosubCheckouts)
	mustDir(wa.gitDirs)
	return &GitTransmat{wa}
}

/*
	Git transmats plonk down the contents of one commit (or tree) as a filesystem.

	A fileset materialized by git does *not* include the `.git` dir by default,
	since those files are not themselves part of what's described by the hash.

	Git effectively "filters" out several attributes -- permissions are only loosely
	respected (execution only), file timestamps are undefined, uid/gid bits
	are not tracked, xattrs are not tracked, etc.  If you desired defined values,
	*you must still configure materialization to use a filter* (particularly for
	file timestamps, since they will otherwise be allowed to vary from one
	materialization to the next(!)).

	Git also allows for several other potential pitfalls with lossless data
	transmission: git cannot transmit empty directories.  This can be a major pain.
	Typical workarounds include creating a ".gitkeep" file in the empty directory.
	Gitignore files may also inadventantly cause trouble.  Transmat.Materialize
	will act *consistently*, but it does not overcome these issues in git
	(doing so would require additional metadata or protocol extensions).
*/
func (t *GitTransmat) Materialize(
	kind rio.TransmatKind,
	dataHash rio.CommitID,
	siloURIs []rio.SiloURI,
	log log15.Logger,
	options ...rio.MaterializerConfigurer,
) rio.Arena {
	var arena gitArena
	meep.Try(func() {
		// Basic validation and config
		mixins.MustBeType(Kind, kind)
{'fraction_non_alphanumeric': 0.09144893111638955, 'fraction_numerical': 0.001583531274742676, 'mean_word_length': 4.456803455723542, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3208285', 'n_tokens_mistral': 1730, 'n_tokens_neox': 1634, 'n_words': 554}