<commit_before>package vfsswift

import (
	"errors"

	multierror "github.com/hashicorp/go-multierror"
	"github.com/ncw/swift"
)

// maxNbFilesToDelete is the maximal number of files that we will try to delete
// in a single call to swift.
const maxNbFilesToDelete = 8000

// maxSimultaneousCalls is the maximal number of simultaneous calls to Swift to
// delete files in the same container.
const maxSimultaneousCalls = 8

var errFailFast = errors.New("fail fast")

// DeleteContainer removes all the files inside the given container, and then
// deletes it.
func DeleteContainer(c *swift.Connection, container string) error {
	_, _, err := c.Container(container)
	if err == swift.ContainerNotFound {
		return nil
	}
	if err != nil {
		return err
	}
	objectNames, err := c.ObjectNamesAll(container, nil)
	if err != nil {
		return err
	}
	if len(objectNames) > 0 {
		if err = deleteContainerFiles(c, container, objectNames); err != nil {
			return err
		}
	}
	return c.ContainerDelete(container)
}

func deleteContainerFiles(c *swift.Connection, container string, objectNames []string) error {
	nb := 1 + (len(objectNames)-1)/maxNbFilesToDelete
	ch := make(chan error)
<commit_msg>Improve the Swift container deletion (#2143)<commit_after>package vfsswift

import (
	"errors"
	"time"

	multierror "github.com/hashicorp/go-multierror"
	"github.com/ncw/swift"
)

// maxNbFilesToDelete is the maximal number of files that we will try to delete
// in a single call to swift.
const maxNbFilesToDelete = 8000

// maxSimultaneousCalls is the maximal number of simultaneous calls to Swift to
// delete files in the same container.
const maxSimultaneousCalls = 8

var errFailFast = errors.New("fail fast")

// DeleteContainer removes all the files inside the given container, and then
// deletes it.
func DeleteContainer(c *swift.Connection, container string) error {
	_, _, err := c.Container(container)
	if err == swift.ContainerNotFound {
		return nil
	}
	if err != nil {
		return err
	}
	objectNames, err := c.ObjectNamesAll(container, nil)
	if err != nil {
		return err
	}
	if len(objectNames) > 0 {
		if err = deleteContainerFiles(c, container, objectNames); err != nil {
			return err
		}
	}

	// XXX Swift has told us that all the files have been deleted on the bulk
	// delete, but it only means that they have been deleted on one object
	// server (at least). And, when we try to delete the container, Swift can
	// send an error as some container servers will still have objects
	// registered for this container. We will try several times to delete the
	// container to work-around this limitation.
	for i := 0; i < 5; i++ {
		err = c.ContainerDelete(container)
		if err == nil {
			return nil
		}
		time.Sleep(5 * time.Second)
	}
	return err
}

func deleteContainerFiles(c *swift.Connection, container string, objectNames []string) error {
	nb := 1 + (len(objectNames)-1)/maxNbFilesToDelete
	ch := make(chan error)

{'fraction_non_alphanumeric': 0.09492803289924606, 'fraction_numerical': 0.007882111034955449, 'mean_word_length': 3.9058823529411764, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1654440', 'n_tokens_mistral': 949, 'n_tokens_neox': 887, 'n_words': 332}