<commit_before>package commands

import (
	"bytes"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	pruneCmd = &cobra.Command{
		Use:   "prune",
		Short: "Deletes old LFS files from the local store",
		Run:   pruneCommand,
	}
	pruneDryRunArg      bool
	pruneVerboseArg     bool
	pruneVerifyArg      bool
	pruneDoNotVerifyArg bool
)

func pruneCommand(cmd *cobra.Command, args []string) {

	// Guts of this must be re-usable from fetch --prune so just parse & dispatch
	if pruneVerifyArg && pruneDoNotVerifyArg {
		Exit("Cannot specify both --verify-remote and --no-verify-remote")
	}

	verify := !pruneDoNotVerifyArg &&
		(lfs.Config.FetchPruneConfig().PruneVerifyRemoteAlways || pruneVerifyArg)

	prune(verify, pruneDryRunArg, pruneVerboseArg)

}

type PruneProgressType int

const (
	PruneProgressTypeLocal  = PruneProgressType(iota)
	PruneProgressTypeRetain = PruneProgressType(iota)
	PruneProgressTypeVerify = PruneProgressType(iota)
)

// Progress from a sub-task of prune
type PruneProgress struct {
	Type  PruneProgressType
	Count int // Number of items done
}
type PruneProgressChan chan PruneProgress

func prune(verifyRemote, dryRun, verbose bool) {
	localObjects := make([]*lfs.Pointer, 0, 100)
	retainedObjects := lfs.NewStringSetWithCapacity(100)
	var reachableObjects lfs.StringSet
	var taskwait sync.WaitGroup

	// Add all the base funcs to the waitgroup before starting them, in case
	// one completes really fast & hits 0 unexpectedly
	// each main process can Add() to the wg itself if it subdivides the task
	taskwait.Add(4) // 1..4: localObjects, current & recent refs, unpushed, worktree
	if verifyRemote {
		taskwait.Add(1) // 5
	}

	progressChan := make(PruneProgressChan, 100)

	// Collect errors
	errorChan := make(chan error, 10)
	var errorwait sync.WaitGroup
	errorwait.Add(1)
	var taskErrors []error
	go pruneTaskCollectErrors(&taskErrors, errorChan, &errorwait)

	// Populate the single list of local objects
	go pruneTaskGetLocalObjects(&localObjects, progressChan, &taskwait)

	// Now find files to be retained from many sources
	retainChan := make(chan string, 100)

	go pruneTaskGetRetainedCurrentAndRecentRefs(retainChan, errorChan, &taskwait)
	go pruneTaskGetRetainedUnpushed(retainChan, errorChan, &taskwait)
	go pruneTaskGetRetainedWorktree(retainChan, errorChan, &taskwait)
	if verifyRemote {
		reachableObjects = lfs.NewStringSetWithCapacity(100)
		go pruneTaskGetReachableObjects(&reachableObjects, errorChan, &taskwait)
	}

	// Now collect all the retained objects, on separate wait
	var retainwait sync.WaitGroup
	retainwait.Add(1)
	go pruneTaskCollectRetained(&retainedObjects, retainChan, progressChan, &retainwait)

	// Report progress
	var progresswait sync.WaitGroup
	progresswait.Add(1)
	go pruneTaskDisplayProgress(progressChan, &progresswait)

	taskwait.Wait()   // wait for subtasks
	close(retainChan) // triggers retain collector to end now all tasks have
	retainwait.Wait() // make sure all retained objects added

	close(errorChan) // triggers error collector to end now all tasks have
	errorwait.Wait() // make sure all errors have been processed
	pruneCheckErrors(taskErrors)

	prunableObjects := make([]string, 0, len(localObjects)/2)

	// Build list of prunables (also queue for verify at same time if applicable)
	var verifyQueue *lfs.TransferQueue
	var verifiedObjects lfs.StringSet
	var totalSize int64
	var verboseOutput bytes.Buffer
	if verifyRemote && !dryRun {
		lfs.Config.CurrentRemote = lfs.Config.FetchPruneConfig().PruneRemoteName
		// build queue now, no estimates or progress output
		verifyQueue = lfs.NewDownloadCheckQueue(0, 0, true)
		verifiedObjects = lfs.NewStringSetWithCapacity(len(localObjects) / 2)
	}
	for _, pointer := range localObjects {
		if !retainedObjects.Contains(pointer.Oid) {
			prunableObjects = append(prunableObjects, pointer.Oid)
			totalSize += pointer.Size
			if verbose {
				// Save up verbose output for the end, spinner still going
				verboseOutput.WriteString(fmt.Sprintf(" * %v (%v)\n", pointer.Oid, humanizeBytes(pointer.Size)))
			}
			if verifyRemote && !dryRun {
				verifyQueue.Add(lfs.NewDownloadCheckable(&lfs.WrappedPointer{Pointer: pointer}))
			}
		}
	}
	if verifyRemote && !dryRun {
		// this channel is filled with oids for which Check() succeeded & Transfer() was called
		verifyc := verifyQueue.Watch()
		var verifywait sync.WaitGroup
		verifywait.Add(1)
		go func() {
			for oid := range verifyc {
				verifiedObjects.Add(oid)
				progressChan <- PruneProgress{PruneProgressTypeVerify, 1}
			}
			verifywait.Done()
		}()
		verifyQueue.Wait()
		verifywait.Wait()
		close(progressChan) // after verify (uses spinner) but before check
		progresswait.Wait()
		pruneCheckVerified(prunableObjects, reachableObjects, verifiedObjects)
	} else {
		close(progressChan)
		progresswait.Wait()
	}

	if len(prunableObjects) == 0 {
		Print("Nothing to prune")
		return
	}
	if dryRun {
		Print("%d files would be pruned (%v)", len(prunableObjects), humanizeBytes(totalSize))
		if verbose {<commit_msg>Verify if asked even in dry-run, results should match just not delete<commit_after>package commands

import (
	"bytes"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/github/git-lfs/vendor/_nuts/github.com/rubyist/tracerx"

	"github.com/github/git-lfs/git"
	"github.com/github/git-lfs/lfs"
	"github.com/github/git-lfs/vendor/_nuts/github.com/spf13/cobra"
)

var (
	pruneCmd = &cobra.Command{
		Use:   "prune",
		Short: "Deletes old LFS files from the local store",
		Run:   pruneCommand,
	}
	pruneDryRunArg      bool
	pruneVerboseArg     bool
	pruneVerifyArg      bool
	pruneDoNotVerifyArg bool
)

func pruneCommand(cmd *cobra.Command, args []string) {

	// Guts of this must be re-usable from fetch --prune so just parse & dispatch
	if pruneVerifyArg && pruneDoNotVerifyArg {
		Exit("Cannot specify both --verify-remote and --no-verify-remote")
	}

	verify := !pruneDoNotVerifyArg &&
		(lfs.Config.FetchPruneConfig().PruneVerifyRemoteAlways || pruneVerifyArg)

	prune(verify, pruneDryRunArg, pruneVerboseArg)

}

type PruneProgressType int

const (
	PruneProgressTypeLocal  = PruneProgressType(iota)
	PruneProgressTypeRetain = PruneProgressType(iota)
	PruneProgressTypeVerify = PruneProgressType(iota)
)

// Progress from a sub-task of prune
type PruneProgress struct {
	Type  PruneProgressType
	Count int // Number of items done
}
type PruneProgressChan chan PruneProgress

func prune(verifyRemote, dryRun, verbose bool) {
	localObjects := make([]*lfs.Pointer, 0, 100)
	retainedObjects := lfs.NewStringSetWithCapacity(100)
	var reachableObjects lfs.StringSet
	var taskwait sync.WaitGroup

	// Add all the base funcs to the waitgroup before starting them, in case
	// one completes really fast & hits 0 unexpectedly
	// each main process can Add() to the wg itself if it subdivides the task
	taskwait.Add(4) // 1..4: localObjects, current & recent refs, unpushed, worktree
	if verifyRemote {
		taskwait.Add(1) // 5
	}

	progressChan := make(PruneProgressChan, 100)

	// Collect errors
	errorChan := make(chan error, 10)
	var errorwait sync.WaitGroup
	errorwait.Add(1)
	var taskErrors []error
	go pruneTaskCollectErrors(&taskErrors, errorChan, &errorwait)

	// Populate the single list of local objects
	go pruneTaskGetLocalObjects(&localObjects, progressChan, &taskwait)

	// Now find files to be retained from many sources
	retainChan := make(chan string, 100)

	go pruneTaskGetRetainedCurrentAndRecentRefs(retainChan, errorChan, &taskwait)
	go pruneTaskGetRetainedUnpushed(retainChan, errorChan, &taskwait)
	go pruneTaskGetRetainedWorktree(retainChan, errorChan, &taskwait)
	if verifyRemote {
		reachableObjects = lfs.NewStringSetWithCapacity(100)
		go pruneTaskGetReachableObjects(&reachableObjects, errorChan, &taskwait)
	}

	// Now collect all the retained objects, on separate wait
	var retainwait sync.WaitGroup
	retainwait.Add(1)
	go pruneTaskCollectRetained(&retainedObjects, retainChan, progressChan, &retainwait)

	// Report progress
	var progresswait sync.WaitGroup
	progresswait.Add(1)
	go pruneTaskDisplayProgress(progressChan, &progresswait)

	taskwait.Wait()   // wait for subtasks
	close(retainChan) // triggers retain collector to end now all tasks have
	retainwait.Wait() // make sure all retained objects added

	close(errorChan) // triggers error collector to end now all tasks have
	errorwait.Wait() // make sure all errors have been processed
	pruneCheckErrors(taskErrors)

	prunableObjects := make([]string, 0, len(localObjects)/2)

	// Build list of prunables (also queue for verify at same time if applicable)
	var verifyQueue *lfs.TransferQueue
	var verifiedObjects lfs.StringSet
	var totalSize int64
	var verboseOutput bytes.Buffer
	if verifyRemote {
		lfs.Config.CurrentRemote = lfs.Config.FetchPruneConfig().PruneRemoteName
		// build queue now, no estimates or progress output
		verifyQueue = lfs.NewDownloadCheckQueue(0, 0, true)
		verifiedObjects = lfs.NewStringSetWithCapacity(len(localObjects) / 2)
	}
	for _, pointer := range localObjects {
		if !retainedObjects.Contains(pointer.Oid) {
			prunableObjects = append(prunableObjects, pointer.Oid)
			totalSize += pointer.Size
			if verbose {
				// Save up verbose output for the end, spinner still going
				verboseOutput.WriteString(fmt.Sprintf(" * %v (%v)\n", pointer.Oid, humanizeBytes(pointer.Size)))
			}
			if verifyRemote {
				verifyQueue.Add(lfs.NewDownloadCheckable(&lfs.WrappedPointer{Pointer: pointer}))
			}
		}
	}
	if verifyRemote {
		// this channel is filled with oids for which Check() succeeded & Transfer() was called
		verifyc := verifyQueue.Watch()
		var verifywait sync.WaitGroup
		verifywait.Add(1)
		go func() {
			for oid := range verifyc {
				verifiedObjects.Add(oid)
				progressChan <- PruneProgress{PruneProgressTypeVerify, 1}
			}
			verifywait.Done()
		}()
		verifyQueue.Wait()
		verifywait.Wait()
		close(progressChan) // after verify (uses spinner) but before check
		progresswait.Wait()
		pruneCheckVerified(prunableObjects, reachableObjects, verifiedObjects)
	} else {
		close(progressChan)
		progresswait.Wait()
	}

	if len(prunableObjects) == 0 {
		Print("Nothing to prune")
		return
	}
	if dryRun {
		Print("%d files would be pruned (%v)", len(prunableObjects), humanizeBytes(totalSize))
		if verbose {
{'fraction_non_alphanumeric': 0.09624796084828711, 'fraction_numerical': 0.007484886287304481, 'mean_word_length': 4.612277867528271, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4006170', 'n_tokens_mistral': 3642, 'n_tokens_neox': 3347, 'n_words': 900}