<commit_before>package orchestrators

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/camptocamp/bivac/pkg/volume"
	//"github.com/rancher/go-rancher-metadata/metadata"
	"github.com/rancher/go-rancher/v2"
	"golang.org/x/net/websocket"
)

// CattleConfig stores Cattle configuration
type CattleConfig struct {
	URL       string
	AccessKey string
	SecretKey string
}

// CattleOrchestrator implements a container orchestrator for Cattle
type CattleOrchestrator struct {
	config *CattleConfig
	client *client.RancherClient
}

// NewCattleOrchestrator creates a Cattle client
func NewCattleOrchestrator(config *CattleConfig) (o *CattleOrchestrator, err error) {
	o = &CattleOrchestrator{
		config: config,
	}
	o.client, err = client.NewRancherClient(&client.ClientOpts{
		Url:       config.URL,
		AccessKey: config.AccessKey,
		SecretKey: config.SecretKey,
		Timeout:   30 * time.Second,
	})
	if err != nil {
		err = fmt.Errorf("failed to create new Rancher client: %s", err)
	}
	return
}

// GetName returns the orchestrator name
func (*CattleOrchestrator) GetName() string {
	return "cattle"
}

// GetPath returns the backup path
func (*CattleOrchestrator) GetPath(v *volume.Volume) string {
	return v.Hostname
}

// GetVolumes returns the Cattle volumes, inspected and filtered
func (o *CattleOrchestrator) GetVolumes(volumeFilters volume.Filters) (volumes []*volume.Volume, err error) {
	vs, err := o.client.Volume.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"limit": -2,
			"all":   true,
		},
	})
	if err != nil {
		err = fmt.Errorf("failed to list volumes: %s", err)
		return
	}

	var mountpoint string
	var h *client.Host
	for _, v := range vs.Data {
		if len(v.Mounts) < 1 {
			mountpoint = "/data"
		} else {
			mountpoint = v.Mounts[0].Path
		}

		var hostID, hostname string
		var spc *client.StoragePoolCollection
		err = o.rawAPICall("GET", v.Links["storagePools"], "", &spc)
		if err != nil {
			continue
		}

		if len(spc.Data) == 0 {
			continue
		}

		if len(spc.Data[0].HostIds) == 0 {
			continue
		}

		hostID = spc.Data[0].HostIds[0]

		h, err = o.client.Host.ById(hostID)
		if err != nil {
			hostname = hostID
		} else {
			hostname = h.Hostname
		}

		v := &volume.Volume{
			ID:         v.Id,
			Name:       v.Name,
			Mountpoint: mountpoint,
			HostBind:   hostID,
			Hostname:   hostname,
		}

		if b, _, _ := o.blacklistedVolume(v, volumeFilters); b {
			continue
		}
		volumes = append(volumes, v)
	}
	return
}

func createAgentName() string {
	var letter = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
	b := make([]rune, 10)
	for i := range b {
		b[i] = letter[rand.Intn(len(letter))]
	}
	return "bivac-agent-" + string(b)
}

// DeployAgent creates a `bivac agent` container
func (o *CattleOrchestrator) DeployAgent(image string, cmd []string, envs []string, v *volume.Volume) (success bool, output string, err error) {
	success = false

	environment := make(map[string]interface{})
	for _, env := range envs {
		splitted := strings.Split(env, "=")
		environment[splitted[0]] = splitted[1]
	}
	container, err := o.client.Container.Create(&client.Container{
		Name:            createAgentName(),
		RequestedHostId: v.HostBind,
		ImageUuid:       "docker:" + image,
		Command:         cmd,
		Environment:     environment,
		RestartPolicy: &client.RestartPolicy{
			MaximumRetryCount: 1,
			Name:              "on-failure",
		},
		Labels: map[string]interface{}{
			"io.rancher.container.pull_image": "always",
		},
		DataVolumes: []string{
			v.Name + ":" + v.Mountpoint,
		},
	})
	if err != nil {
		err = fmt.Errorf("failed to create an agent container: %s", err)
		return
	}
	defer o.RemoveContainer(container)

	stopped := false
	terminated := false
	for !terminated {
		container, err := o.client.Container.ById(container.Id)
		if err != nil {
			err = fmt.Errorf("failed to inspect agent: %s", err)
			return false, "", err
		}

		// This workaround is awful but it's the only way to know if the container failed.
		if container.State == "stopped" {
			if container.StartCount == 1 {
				if stopped == false {
					stopped = true
					time.Sleep(5 * time.Second)
				} else {
					terminated = true
					success = true
				}
			} else {
				success = false
				terminated = true
			}
		}
	}

	container, err = o.client.Container.ById(container.Id)
	if err != nil {
		err = fmt.Errorf("failed to inspect the agent before retrieving the logs: %s", err)
		return false, "", err
	}

	var hostAccess *client.HostAccess
	logsParams := `{"follow":false,"lines":9999,"since":"","timestamps":true}`
	err = o.rawAPICall("POST", container.Links["self"]+"/?action=logs", logsParams, &hostAccess)
	if err != nil {
		err = fmt.Errorf("failed to read response from rancher: %s", err)
		return
	}

	origin := o.config.URL

	u, err := url.Parse(hostAccess.Url)
	if err != nil {
		err = fmt.Errorf("failed to parse rancher server url: %s", err)
		return<commit_msg>timeout after 30s on container creation<commit_after>package orchestrators

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/camptocamp/bivac/pkg/volume"
	//"github.com/rancher/go-rancher-metadata/metadata"
	"github.com/rancher/go-rancher/v2"
	"golang.org/x/net/websocket"
)

// CattleConfig stores Cattle configuration
type CattleConfig struct {
	URL       string
	AccessKey string
	SecretKey string
}

// CattleOrchestrator implements a container orchestrator for Cattle
type CattleOrchestrator struct {
	config *CattleConfig
	client *client.RancherClient
}

// NewCattleOrchestrator creates a Cattle client
func NewCattleOrchestrator(config *CattleConfig) (o *CattleOrchestrator, err error) {
	o = &CattleOrchestrator{
		config: config,
	}
	o.client, err = client.NewRancherClient(&client.ClientOpts{
		Url:       config.URL,
		AccessKey: config.AccessKey,
		SecretKey: config.SecretKey,
		Timeout:   30 * time.Second,
	})
	if err != nil {
		err = fmt.Errorf("failed to create new Rancher client: %s", err)
	}
	return
}

// GetName returns the orchestrator name
func (*CattleOrchestrator) GetName() string {
	return "cattle"
}

// GetPath returns the backup path
func (*CattleOrchestrator) GetPath(v *volume.Volume) string {
	return v.Hostname
}

// GetVolumes returns the Cattle volumes, inspected and filtered
func (o *CattleOrchestrator) GetVolumes(volumeFilters volume.Filters) (volumes []*volume.Volume, err error) {
	vs, err := o.client.Volume.List(&client.ListOpts{
		Filters: map[string]interface{}{
			"limit": -2,
			"all":   true,
		},
	})
	if err != nil {
		err = fmt.Errorf("failed to list volumes: %s", err)
		return
	}

	var mountpoint string
	var h *client.Host
	for _, v := range vs.Data {
		if len(v.Mounts) < 1 {
			mountpoint = "/data"
		} else {
			mountpoint = v.Mounts[0].Path
		}

		var hostID, hostname string
		var spc *client.StoragePoolCollection
		err = o.rawAPICall("GET", v.Links["storagePools"], "", &spc)
		if err != nil {
			continue
		}

		if len(spc.Data) == 0 {
			continue
		}

		if len(spc.Data[0].HostIds) == 0 {
			continue
		}

		hostID = spc.Data[0].HostIds[0]

		h, err = o.client.Host.ById(hostID)
		if err != nil {
			hostname = hostID
		} else {
			hostname = h.Hostname
		}

		v := &volume.Volume{
			ID:         v.Id,
			Name:       v.Name,
			Mountpoint: mountpoint,
			HostBind:   hostID,
			Hostname:   hostname,
		}

		if b, _, _ := o.blacklistedVolume(v, volumeFilters); b {
			continue
		}
		volumes = append(volumes, v)
	}
	return
}

func createAgentName() string {
	var letter = []rune("abcdefghijklmnopqrstuvwxyz0123456789")
	b := make([]rune, 10)
	for i := range b {
		b[i] = letter[rand.Intn(len(letter))]
	}
	return "bivac-agent-" + string(b)
}

// DeployAgent creates a `bivac agent` container
func (o *CattleOrchestrator) DeployAgent(image string, cmd []string, envs []string, v *volume.Volume) (success bool, output string, err error) {
	success = false

	environment := make(map[string]interface{})
	for _, env := range envs {
		splitted := strings.Split(env, "=")
		environment[splitted[0]] = splitted[1]
	}
	container, err := o.client.Container.Create(&client.Container{
		Name:            createAgentName(),
		RequestedHostId: v.HostBind,
		ImageUuid:       "docker:" + image,
		Command:         cmd,
		Environment:     environment,
		RestartPolicy: &client.RestartPolicy{
			MaximumRetryCount: 1,
			Name:              "on-failure",
		},
		Labels: map[string]interface{}{
			"io.rancher.container.pull_image": "always",
		},
		DataVolumes: []string{
			v.Name + ":" + v.Mountpoint,
		},
	})
	if err != nil {
		err = fmt.Errorf("failed to create an agent container: %s", err)
		return
	}
	defer o.RemoveContainer(container)

	stopped := false
	terminated := false
	timeoutCount := 0
	for !terminated && timeoutCount <= 30 {
		container, err := o.client.Container.ById(container.Id)
		if err != nil {
			err = fmt.Errorf("failed to inspect agent: %s", err)
			return false, "", err
		}

		// This workaround is awful but it's the only way to know if the container failed.
		if container.State == "stopped" {
			if container.StartCount == 1 {
				if stopped == false {
					stopped = true
					time.Sleep(5 * time.Second)
				} else {
					terminated = true
					success = true
				}
			} else {
				success = false
				terminated = true
			}
		} else if container.State == "starting" {
			timeoutCount++
		}
		time.Sleep(1 * time.Second)
	}

	if timeoutCount == 30 {
		success = false
		err = fmt.Errorf("failed to start container: timeout")
		return
	}

	container, err = o.client.Container.ById(container.Id)
	if err != nil {
		err = fmt.Errorf("failed to inspect the agent before retrieving the logs: %s", err)
		return false, "", err
	}

	var hostAccess *client.HostAccess
	logsParams := `{"follow":false,"lines":9999,"since":"","timestamps":true}`
	err = o.rawAPICall("POST", container.Links["self"]+"/?action=logs", logsParams, &hostAccess)
	if err != nil {
		err = fmt.Errorf("failed to read response from rancher: %s", err)
		return
	}

	origin := o.config.URL

	u, err := url.Parse(hostAccess.Url)
	if err != nil {
		err = fmt.Errorf("failed to parse rancher server url: %s", err)
		return
{'fraction_non_alphanumeric': 0.13763773439010246, 'fraction_numerical': 0.007539145563502803, 'mean_word_length': 3.4484092863284608, 'pattern_counts': {'":': 16, '<': 6, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7191903', 'n_tokens_mistral': 4031, 'n_tokens_neox': 3667, 'n_words': 915}