<commit_before>package gcng

import (
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/dbng"
	"github.com/concourse/atc/worker/transport"
	"github.com/concourse/baggageclaim"
	bclient "github.com/concourse/baggageclaim/client"
)

type volumeCollector struct {
	rootLogger                lager.Logger
	volumeFactory             dbng.VolumeFactory
	baggageclaimClientFactory BaggageclaimClientFactory
}

//go:generate counterfeiter . BaggageclaimClientFactory

type BaggageclaimClientFactory interface {
	NewClient(apiURL string, workerName string) bclient.Client
}

type baggageclaimClientFactory struct {
	dbWorkerFactory dbng.WorkerFactory
}

func NewBaggageclaimClientFactory(dbWorkerFactory dbng.WorkerFactory) BaggageclaimClientFactory {
	return &baggageclaimClientFactory{
		dbWorkerFactory: dbWorkerFactory,
	}
}

func (f *baggageclaimClientFactory) NewClient(apiURL string, workerName string) bclient.Client {
	rountTripper := transport.NewBaggageclaimRoundTripper(
		workerName,
		&apiURL,
		f.dbWorkerFactory,
		&http.Transport{DisableKeepAlives: true},
	)
	return bclient.New(apiURL, rountTripper)
}

func NewVolumeCollector(
	logger lager.Logger,
	volumeFactory dbng.VolumeFactory,
	baggageclaimClientFactory BaggageclaimClientFactory,
) Collector {
	return &volumeCollector{
		rootLogger:                logger,
		volumeFactory:             volumeFactory,
		baggageclaimClientFactory: baggageclaimClientFactory,
	}
}

func (vc *volumeCollector) Run() error {
	logger := vc.rootLogger.Session("run")

	logger.Debug("start")
	defer logger.Debug("done")

	var (
		creatingVolumes   []dbng.CreatingVolume
		createdVolumes    []dbng.CreatedVolume
		destroyingVolumes []dbng.DestroyingVolume
		err               error
	)

	creatingVolumes, createdVolumes, destroyingVolumes, err = vc.volumeFactory.GetDuplicateResourceCacheVolumes()
	if err != nil {
		logger.Error("failed-to-get-duplicate-resource-cache-volumes", err)
		return err
	}

	orphanedCreatedVolumes, orphanedDestroyingVolumes, err := vc.volumeFactory.GetOrphanedVolumes()
	if err != nil {
		logger.Error("failed-to-get-orphaned-volumes", err)
		return err
	}

	createdVolumes = append(createdVolumes, orphanedCreatedVolumes...)
	destroyingVolumes = append(destroyingVolumes, orphanedDestroyingVolumes...)

	for _, creatingVolume := range creatingVolumes {
		vLog := logger.Session("mark-creating-as-created", lager.Data{
			"volume": creatingVolume.Handle(),
		})

		createdVolume, err := creatingVolume.Created()
		if err != nil {
			vLog.Error("failed-to-transition-from-creating-to-created", err)
			continue
		}

		createdVolumes = append(createdVolumes, createdVolume)
	}

	for _, createdVolume := range createdVolumes {
		vLog := logger.Session("mark-created-as-destroying", lager.Data{
			"volume": createdVolume.Handle(),
			"worker": createdVolume.Worker().Name,
		})

		destroyingVolume, err := createdVolume.Destroying()
		if err != nil {
			vLog.Error("failed-to-transition", err)
			continue
		}
<commit_msg>log how many orphans to kill<commit_after>package gcng

import (
	"net/http"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc/dbng"
	"github.com/concourse/atc/worker/transport"
	"github.com/concourse/baggageclaim"
	bclient "github.com/concourse/baggageclaim/client"
)

type volumeCollector struct {
	rootLogger                lager.Logger
	volumeFactory             dbng.VolumeFactory
	baggageclaimClientFactory BaggageclaimClientFactory
}

//go:generate counterfeiter . BaggageclaimClientFactory

type BaggageclaimClientFactory interface {
	NewClient(apiURL string, workerName string) bclient.Client
}

type baggageclaimClientFactory struct {
	dbWorkerFactory dbng.WorkerFactory
}

func NewBaggageclaimClientFactory(dbWorkerFactory dbng.WorkerFactory) BaggageclaimClientFactory {
	return &baggageclaimClientFactory{
		dbWorkerFactory: dbWorkerFactory,
	}
}

func (f *baggageclaimClientFactory) NewClient(apiURL string, workerName string) bclient.Client {
	rountTripper := transport.NewBaggageclaimRoundTripper(
		workerName,
		&apiURL,
		f.dbWorkerFactory,
		&http.Transport{DisableKeepAlives: true},
	)
	return bclient.New(apiURL, rountTripper)
}

func NewVolumeCollector(
	logger lager.Logger,
	volumeFactory dbng.VolumeFactory,
	baggageclaimClientFactory BaggageclaimClientFactory,
) Collector {
	return &volumeCollector{
		rootLogger:                logger,
		volumeFactory:             volumeFactory,
		baggageclaimClientFactory: baggageclaimClientFactory,
	}
}

func (vc *volumeCollector) Run() error {
	logger := vc.rootLogger.Session("run")

	logger.Debug("start")
	defer logger.Debug("done")

	var (
		creatingVolumes   []dbng.CreatingVolume
		createdVolumes    []dbng.CreatedVolume
		destroyingVolumes []dbng.DestroyingVolume
		err               error
	)

	creatingVolumes, createdVolumes, destroyingVolumes, err = vc.volumeFactory.GetDuplicateResourceCacheVolumes()
	if err != nil {
		logger.Error("failed-to-get-duplicate-resource-cache-volumes", err)
		return err
	}

	if len(creatingVolumes) > 0 || len(createdVolumes) > 0 || len(destroyingVolumes) > 0 {
		logger.Debug("found-duplicate-resource-cache-volumes", lager.Data{
			"creating":   len(creatingVolumes),
			"created":    len(createdVolumes),
			"destroying": len(destroyingVolumes),
		})
	}

	orphanedCreatedVolumes, orphanedDestroyingVolumes, err := vc.volumeFactory.GetOrphanedVolumes()
	if err != nil {
		logger.Error("failed-to-get-orphaned-volumes", err)
		return err
	}

	if len(orphanedCreatedVolumes) > 0 || len(orphanedDestroyingVolumes) > 0 {
		logger.Debug("found-orphaned-volumes", lager.Data{
			"created":    len(createdVolumes),
			"destroying": len(destroyingVolumes),
		})
	}

	createdVolumes = append(createdVolumes, orphanedCreatedVolumes...)
	destroyingVolumes = append(destroyingVolumes, orphanedDestroyingVolumes...)

	for _, creatingVolume := range creatingVolumes {
		vLog := logger.Session("mark-creating-as-created", lager.Data{
			"volume": creatingVolume.Handle(),
		})

		createdVolume, err := creatingVolume.Created()
		if err != nil {
			vLog.Error("failed-to-transition-from-creating-to-created", err)
			continue
		}

		createdVolumes = append(createdVolumes, createdVolume)
	}

	for _, createdVolume := range createdVolumes {
		vLog := logger.Session("mark-created-as-destroying", lager.Data{
			"volume": createdVolume.Handle(),
			"worker": createdVolume.Worker().Name,
		})

		destroyingVolume, err := createdVolume.Destroying()
		if err != nil {
			vLog.Error("failed-to-transition", err)
			continue
		}

{'fraction_non_alphanumeric': 0.10660752523707556, 'fraction_numerical': 0.0007647598654022637, 'mean_word_length': 4.812444444444444, 'pattern_counts': {'":': 11, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5864960', 'n_tokens_mistral': 2245, 'n_tokens_neox': 2069, 'n_words': 365}