<commit_before>package locks

import (
	"encoding/hex"
	"fmt"
	"time"

	etcd "github.com/coreos/etcd/client"
	"golang.org/x/net/context"
)

const BUILDLOCKS = "/buildlocks"
const DEFERRED = "/deferred"

type Locker interface {
	Lock(key, value string) (*etcd.Response, error)
	Unlock(key, value string) (*etcd.Response, error)
	Defer(buildEvent []byte) (*etcd.Response, error)
	ClearDeferred(deferredID string) (*etcd.Response, error)
	DeferredBuilds() ([]Deferral, error)
	InitDeferred() error
	Key(projectKey, branch string) string
}

type Deferral struct {
	Data string `json:"-"`
	Key  string `json:"key"`
}

type EtcdLocker struct {
	Config etcd.Config
}

func (d EtcdLocker) Lock(key, value string) (*etcd.Response, error) {
	c, err := etcd.New(d.Config)
	if err != nil {
		return nil, err
	}
	return etcd.NewKeysAPI(c).Set(context.Background(), BUILDLOCKS+"/"+key, value, &etcd.SetOptions{PrevExist: etcd.PrevNoExist})
}

func (d EtcdLocker) Unlock(key, value string) (*etcd.Response, error) {
	c, err := etcd.New(d.Config)
	if err != nil {
		return nil, err
	}
	return etcd.NewKeysAPI(c).Delete(context.Background(), BUILDLOCKS+"/"+key, &etcd.DeleteOptions{PrevValue: value})
}

func (d EtcdLocker) Key(projectKey, branch string) string {
	return hex.EncodeToString([]byte(fmt.Sprintf("%s/%s", projectKey, branch)))
}

func (d EtcdLocker) Defer(buildEvent []byte) (*etcd.Response, error) {
	c, err := etcd.New(d.Config)
	if err != nil {
		return nil, err
	}
	return etcd.NewKeysAPI(c).CreateInOrder(context.Background(), DEFERRED, string(buildEvent), nil)
}

func (d EtcdLocker) ClearDeferred(deferralID string) (*etcd.Response, error) {
	c, err := etcd.New(d.Config)
	if err != nil {
		return nil, err
	}
	return etcd.NewKeysAPI(c).Delete(context.Background(), deferralID, nil)
}

func (d EtcdLocker) DeferredBuilds() ([]Deferral, error) {
	c, err := etcd.New(d.Config)
	if err != nil {
		return nil, err
	}
	resp, err := etcd.NewKeysAPI(c).Get(context.Background(), DEFERRED, &etcd.GetOptions{Recursive: true, Sort: true})
	if err != nil {
		return nil, err
	}

	events := make([]Deferral, 0)
	for _, v := range resp.Node.Nodes {
		d := Deferral{Data: v.Value, Key: v.Key}
		events = append(events, d)
	}
	return events, nil
}

// Create the deferred directory in etcd
func (d EtcdLocker) InitDeferred() error {
	c, err := etcd.New(d.Config)
	if err != nil {
		return err
	}
	_, err = etcd.NewKeysAPI(c).Set(context.Background(), DEFERRED, "", &etcd.SetOptions{Dir: true})
	return err
}

func NewEtcdLocker(machines []string) Locker {
	return EtcdLocker{Config: etcd.Config{
		Endpoints: machines,
		Transport: etcd.DefaultTransport,
		// set timeout per request to fail fast when the target endpoint is unavailable
		HeaderTimeoutPerRequest: time.Second,
	},
	}
}
<commit_msg>if /deferred exists in etcd, do not attempt to create it again<commit_after>package locks

import (
	"encoding/hex"
	"fmt"
	"time"

	etcd "github.com/coreos/etcd/client"
	"golang.org/x/net/context"
)

const BUILDLOCKS = "/buildlocks"
const DEFERRED = "/deferred"

type Locker interface {
	Lock(key, value string) (*etcd.Response, error)
	Unlock(key, value string) (*etcd.Response, error)
	Defer(buildEvent []byte) (*etcd.Response, error)
	ClearDeferred(deferredID string) (*etcd.Response, error)
	DeferredBuilds() ([]Deferral, error)
	InitDeferred() error
	Key(projectKey, branch string) string
}

type Deferral struct {
	Data string `json:"-"`
	Key  string `json:"key"`
}

type EtcdLocker struct {
	Config etcd.Config
}

func (d EtcdLocker) Lock(key, value string) (*etcd.Response, error) {
	c, err := etcd.New(d.Config)
	if err != nil {
		return nil, err
	}
	return etcd.NewKeysAPI(c).Set(context.Background(), BUILDLOCKS+"/"+key, value, &etcd.SetOptions{PrevExist: etcd.PrevNoExist})
}

func (d EtcdLocker) Unlock(key, value string) (*etcd.Response, error) {
	c, err := etcd.New(d.Config)
	if err != nil {
		return nil, err
	}
	return etcd.NewKeysAPI(c).Delete(context.Background(), BUILDLOCKS+"/"+key, &etcd.DeleteOptions{PrevValue: value})
}

func (d EtcdLocker) Key(projectKey, branch string) string {
	return hex.EncodeToString([]byte(fmt.Sprintf("%s/%s", projectKey, branch)))
}

func (d EtcdLocker) Defer(buildEvent []byte) (*etcd.Response, error) {
	c, err := etcd.New(d.Config)
	if err != nil {
		return nil, err
	}
	return etcd.NewKeysAPI(c).CreateInOrder(context.Background(), DEFERRED, string(buildEvent), nil)
}

func (d EtcdLocker) ClearDeferred(deferralID string) (*etcd.Response, error) {
	c, err := etcd.New(d.Config)
	if err != nil {
		return nil, err
	}
	return etcd.NewKeysAPI(c).Delete(context.Background(), deferralID, nil)
}

func (d EtcdLocker) DeferredBuilds() ([]Deferral, error) {
	c, err := etcd.New(d.Config)
	if err != nil {
		return nil, err
	}
	resp, err := etcd.NewKeysAPI(c).Get(context.Background(), DEFERRED, &etcd.GetOptions{Recursive: true, Sort: true})
	if err != nil {
		return nil, err
	}

	events := make([]Deferral, 0)
	for _, v := range resp.Node.Nodes {
		d := Deferral{Data: v.Value, Key: v.Key}
		events = append(events, d)
	}
	return events, nil
}

// Create the deferred directory in etcd.  Check whether a listing is available; if one is, do not create the directory again.
func (d EtcdLocker) InitDeferred() error {
	_, err := d.DeferredBuilds()
	if err != nil {
		var c etcd.Client
		c, err = etcd.New(d.Config)
		if err != nil {
			return err
		}
		_, err = etcd.NewKeysAPI(c).Set(context.Background(), DEFERRED, "", &etcd.SetOptions{Dir: true})
	}
	return err
}

func NewEtcdLocker(machines []string) Locker {
	return EtcdLocker{Config: etcd.Config{
		Endpoints: machines,
		Transport: etcd.DefaultTransport,
		// set timeout per request to fail fast when the target endpoint is unavailable
		HeaderTimeoutPerRequest: time.Second,
	},
	}
}

{'fraction_non_alphanumeric': 0.14905758256960056, 'fraction_numerical': 0.0003458412588621823, 'mean_word_length': 4.615533980582525, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '466136', 'n_tokens_mistral': 2184, 'n_tokens_neox': 2125, 'n_words': 544}