<commit_before>// Copyright 2014-2015 The DevMine authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package repo

import (
	"archive/tar"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	g2g "github.com/libgit2/git2go"

	"github.com/DevMine/repotool/config"
	"github.com/DevMine/repotool/model"
)

// gitRepo is a repository with some things specific to git.
type gitRepo struct {
	model.Repository
	cfg    config.DataConfig
	r      *g2g.Repository
	tmpDir string
}

// New creates a new gitRepo object.
func newGitRepo(cfg config.DataConfig, repository model.Repository, gitDir string, useTmpDir bool) (*gitRepo, error) {
	r, err := g2g.OpenRepository(gitDir)
	if err != nil {
		return nil, err
	}

	var tmpDir string
	if useTmpDir {
		tmpDir = gitDir
	}

	return &gitRepo{Repository: repository, cfg: cfg, r: r, tmpDir: tmpDir}, nil
}

// FetchCommits fetches all commits from a Git repository and adds them to
// the list of commits of the repository object.
func (gr *gitRepo) FetchCommits() error {
	gr.Commits = make([]model.Commit, 0) // give number of commits

	rw, err := gr.r.Walk()
	if err != nil {
		return err
	}

	err = rw.PushHead()
	if err != nil {
		return err
	}

	err = rw.Iterate(gr.addCommit)
	if err != nil {
		return err
	}

	return nil
}

// GetRepository returns the repository structre contained in a git repository.
func (gr gitRepo) GetRepository() *model.Repository {
	return &gr.Repository
}

// GetName returns the name of a git repository.
func (gr gitRepo) GetName() string {
	return gr.Name
}

// GetVCS returns the VCS type (shall be "git").
func (gr gitRepo) GetVCS() string {
	return gr.VCS
}

// GetCloneURL returns the git repository clone URL.
func (gr gitRepo) GetCloneURL() string {
	return gr.CloneURL
}

// GetClonePath returns the clone path of a git repository.
func (gr gitRepo) GetClonePath() string {
	return gr.ClonePath
}

// GetDefaultBranch returns the git repository default branch.
func (gr gitRepo) GetDefaultBranch() string {
	return gr.DefaultBranch
}

// GetCommits returns the list of commits in the git repository.
// If the list is empty of nil, this probably means that a call to
// FetchCommits() is needed to populate the list.
func (gr gitRepo) GetCommits() []model.Commit {
	return gr.Commits
}

// Cleanup frees open repositories and removes temporary created files, if any.
func (gr gitRepo) Cleanup() error {
	if gr.r != nil {
		gr.r.Free()
	}

	if len(gr.tmpDir) > 0 {
		return os.RemoveAll(gr.tmpDir)
	}
	return nil
}

var deltaMap = map[g2g.Delta]*string{
	g2g.DeltaUnmodified: nil,
	g2g.DeltaAdded:      &model.StatusAdded,
	g2g.DeltaDeleted:    &model.StatusDeleted,
	g2g.DeltaModified:   &model.StatusModified,
	g2g.DeltaRenamed:    &model.StatusRenamed,
	g2g.DeltaCopied:     &model.StatusCopied,
	g2g.DeltaIgnored:    nil,
	g2g.DeltaUntracked:  nil,
	g2g.DeltaTypeChange: nil,
}

// addCommit is conform to the g2g.RevWalIterator type in order to be used
// by the g2g.Iterate() function to iterate over all commits of a Git repository.
func (gr *gitRepo) addCommit(c *g2g.Commit) bool {
	if c == nil {
		return false
	}

	var commit model.Commit

	oID := c.Id()
	if oID == nil {
		return false
	}
	commit.VCSID = oID.String()

	commit.Message = c.Message()

	var author model.Developer
	author.Name = c.Author().Name
	author.Email = c.Author().Email
	commit.Author = author

	var committer model.Developer
	committer.Name = c.Committer().Name
	committer.Email = c.Committer().Email
	commit.Committer = committer

	commit.CommitDate = c.Committer().When
	commit.AuthorDate = c.Author().When

	parentC := c.Parent(0)
	if parentC == nil {
		return false
	}

	parentTree, err := parentC.Tree()
	if err != nil {
		return false
	}

	cTree, err := c.Tree()
	if err != nil {
		return false
	}

	diffOpts, err := g2g.DefaultDiffOptions()
	if err != nil {
		return false
	}

	diff, err := gr.r.DiffTreeToTree(parentTree, cTree, &diffOpts)
	if err != nil {
		return false
	}

	stats, err := diff.Stats()
	if err != nil {
		return false
	}

	nDeltas, err := diff.NumDeltas()
	if err != nil {
		return false
	}

	if gr.cfg.CommitDeltas {
		for d := 0; d < nDeltas; d++ {
			var cdd model.DiffDelta

			if gr.cfg.CommitPatches {
				patch, err := diff.Patch(d)
				if err != nil || patch == nil {
					return false
				}
				p, err := patch.String()
				if err != nil {
					return false
				}
				cdd.Patch = &p
			}

			diffDelta, err := diff.GetDelta(d)
			if err != nil {
				return false
			}
			cdd.Status = deltaMap[diffDelta.Status]<commit_msg>repo/git: numDeltas is only needed when using deltas<commit_after>// Copyright 2014-2015 The DevMine authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package repo

import (
	"archive/tar"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	g2g "github.com/libgit2/git2go"

	"github.com/DevMine/repotool/config"
	"github.com/DevMine/repotool/model"
)

// gitRepo is a repository with some things specific to git.
type gitRepo struct {
	model.Repository
	cfg    config.DataConfig
	r      *g2g.Repository
	tmpDir string
}

// New creates a new gitRepo object.
func newGitRepo(cfg config.DataConfig, repository model.Repository, gitDir string, useTmpDir bool) (*gitRepo, error) {
	r, err := g2g.OpenRepository(gitDir)
	if err != nil {
		return nil, err
	}

	var tmpDir string
	if useTmpDir {
		tmpDir = gitDir
	}

	return &gitRepo{Repository: repository, cfg: cfg, r: r, tmpDir: tmpDir}, nil
}

// FetchCommits fetches all commits from a Git repository and adds them to
// the list of commits of the repository object.
func (gr *gitRepo) FetchCommits() error {
	gr.Commits = make([]model.Commit, 0) // give number of commits

	rw, err := gr.r.Walk()
	if err != nil {
		return err
	}

	err = rw.PushHead()
	if err != nil {
		return err
	}

	err = rw.Iterate(gr.addCommit)
	if err != nil {
		return err
	}

	return nil
}

// GetRepository returns the repository structre contained in a git repository.
func (gr gitRepo) GetRepository() *model.Repository {
	return &gr.Repository
}

// GetName returns the name of a git repository.
func (gr gitRepo) GetName() string {
	return gr.Name
}

// GetVCS returns the VCS type (shall be "git").
func (gr gitRepo) GetVCS() string {
	return gr.VCS
}

// GetCloneURL returns the git repository clone URL.
func (gr gitRepo) GetCloneURL() string {
	return gr.CloneURL
}

// GetClonePath returns the clone path of a git repository.
func (gr gitRepo) GetClonePath() string {
	return gr.ClonePath
}

// GetDefaultBranch returns the git repository default branch.
func (gr gitRepo) GetDefaultBranch() string {
	return gr.DefaultBranch
}

// GetCommits returns the list of commits in the git repository.
// If the list is empty of nil, this probably means that a call to
// FetchCommits() is needed to populate the list.
func (gr gitRepo) GetCommits() []model.Commit {
	return gr.Commits
}

// Cleanup frees open repositories and removes temporary created files, if any.
func (gr gitRepo) Cleanup() error {
	if gr.r != nil {
		gr.r.Free()
	}

	if len(gr.tmpDir) > 0 {
		return os.RemoveAll(gr.tmpDir)
	}
	return nil
}

var deltaMap = map[g2g.Delta]*string{
	g2g.DeltaUnmodified: nil,
	g2g.DeltaAdded:      &model.StatusAdded,
	g2g.DeltaDeleted:    &model.StatusDeleted,
	g2g.DeltaModified:   &model.StatusModified,
	g2g.DeltaRenamed:    &model.StatusRenamed,
	g2g.DeltaCopied:     &model.StatusCopied,
	g2g.DeltaIgnored:    nil,
	g2g.DeltaUntracked:  nil,
	g2g.DeltaTypeChange: nil,
}

// addCommit is conform to the g2g.RevWalIterator type in order to be used
// by the g2g.Iterate() function to iterate over all commits of a Git repository.
func (gr *gitRepo) addCommit(c *g2g.Commit) bool {
	if c == nil {
		return false
	}

	var commit model.Commit

	oID := c.Id()
	if oID == nil {
		return false
	}
	commit.VCSID = oID.String()

	commit.Message = c.Message()

	var author model.Developer
	author.Name = c.Author().Name
	author.Email = c.Author().Email
	commit.Author = author

	var committer model.Developer
	committer.Name = c.Committer().Name
	committer.Email = c.Committer().Email
	commit.Committer = committer

	commit.CommitDate = c.Committer().When
	commit.AuthorDate = c.Author().When

	parentC := c.Parent(0)
	if parentC == nil {
		return false
	}

	parentTree, err := parentC.Tree()
	if err != nil {
		return false
	}

	cTree, err := c.Tree()
	if err != nil {
		return false
	}

	diffOpts, err := g2g.DefaultDiffOptions()
	if err != nil {
		return false
	}

	diff, err := gr.r.DiffTreeToTree(parentTree, cTree, &diffOpts)
	if err != nil {
		return false
	}

	stats, err := diff.Stats()
	if err != nil {
		return false
	}

	if gr.cfg.CommitDeltas {
		nDeltas, err := diff.NumDeltas()
		if err != nil {
			return false
		}

		for d := 0; d < nDeltas; d++ {
			var cdd model.DiffDelta

			if gr.cfg.CommitPatches {
				patch, err := diff.Patch(d)
				if err != nil || patch == nil {
					return false
				}
				p, err := patch.String()
				if err != nil {
					return false
				}
				cdd.Patch = &p
			}

			diffDelta, err := diff.GetDelta(d)
			if err != nil {
				return false
			}
			cdd.Status = deltaMap[diffDelta.Status]
{'fraction_non_alphanumeric': 0.11356059791375417, 'fraction_numerical': 0.006667383589633294, 'mean_word_length': 3.303563165201296, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7165609', 'n_tokens_mistral': 3479, 'n_tokens_neox': 3320, 'n_words': 1028}