<commit_before>package main

import (
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"
	"webup/backoops/config"
	"webup/backoops/options"
	"webup/backoops/services"

	log "github.com/Sirupsen/logrus"
	"github.com/jawher/mow.cli"
	"github.com/ncw/swift"

	"golang.org/x/net/context"
)

const (
	swiftContainerName = "backups"
	etcdRootDir        = "/backups"
)

type backupInfo struct {
	Name   string
	Expire time.Time
	URL    string
}

func (info backupInfo) String() string {
	return fmt.Sprintf("    name: %s\n", info.Name) +
		fmt.Sprintf(" expires: %v\n", info.Expire) +
		fmt.Sprintf("     url: %s\n", info.URL)
}

func main() {
	app := cli.App("backoops", "Perform backups")

	swiftURL := app.String(cli.StringOpt{
		Name:   "swift-auth-url",
		Value:  "",
		Desc:   "Swift auth URL",
		EnvVar: "OS_AUTH_URL",
	})
	swiftUser := app.String(cli.StringOpt{
		Name:   "swift-user",
		Value:  "",
		Desc:   "Swift username",
		EnvVar: "OS_USERNAME",
	})
	swiftAPIKey := app.String(cli.StringOpt{
		Name:   "swift-password",
		Value:  "",
		Desc:   "Swift API Key / Password",
		EnvVar: "OS_PASSWORD",
	})
	swiftTenantName := app.String(cli.StringOpt{
		Name:   "swift-tenant-name",
		Value:  "",
		Desc:   "Swift Tenant name",
		EnvVar: "OS_TENANT_NAME",
	})

	app.Command("daemon", "Start the backup process", func(cmd *cli.Cmd) {

		cmd.Spec = "-w... [--etcd]"

		etcdEndpoints := getEtcdOptionsFromCli(cmd)

		watchDirs := cmd.StringsOpt("w watch", []string{}, "Specifies the directories to watch for finding backup.yml files")

		cmd.Action = func() {

			ctx, cancel := context.WithCancel(context.Background())

			// prepare options
			opts := options.NewDefaultOptions()
			opts.EtcdEndpoints = strings.Split(*etcdEndpoints, ",")
			opts.WatchDirs = *watchDirs
			opts.Swift = options.SwiftOptions{
				AuthURL:       *swiftURL,
				User:          *swiftUser,
				APIKey:        *swiftAPIKey,
				TenantName:    *swiftTenantName,
				ContainerName: swiftContainerName,
			}

			ctx = options.NewContext(ctx, opts)

			// handle the SIGINT signal
			waiting := make(chan os.Signal, 1)
			signal.Notify(waiting, os.Interrupt)

			// store the state of the running backups (by project)
			runningState := make(chan map[string]bool)

			// start backup fetching daemon
			go services.FetchBackupConfig(ctx, runningState)
			// start backup routine
			go services.PerformBackup(ctx, runningState)

			// waiting for signal
			<-waiting

			// cancelling ctx
			cancel()

			fmt.Println("\n Exiting.")
		}
	})

	app.Command("get", "List the available backup archives for a project", func(cmd *cli.Cmd) {

		cmd.Spec = "NAME"

		name := cmd.StringArg("NAME", "", "The name of the project")

		cmd.Action = func() {

			fmt.Println("Searching...")

			swiftOptions := options.SwiftOptions{
				AuthURL:       *swiftURL,
				User:          *swiftUser,<commit_msg>Some comments to remember if concurrency handling can be improved<commit_after>package main

import (
	"fmt"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"
	"webup/backoops/config"
	"webup/backoops/options"
	"webup/backoops/services"

	log "github.com/Sirupsen/logrus"
	"github.com/jawher/mow.cli"
	"github.com/ncw/swift"

	"golang.org/x/net/context"
)

const (
	swiftContainerName = "backups"
	etcdRootDir        = "/backups"
)

type backupInfo struct {
	Name   string
	Expire time.Time
	URL    string
}

func (info backupInfo) String() string {
	return fmt.Sprintf("    name: %s\n", info.Name) +
		fmt.Sprintf(" expires: %v\n", info.Expire) +
		fmt.Sprintf("     url: %s\n", info.URL)
}

func main() {
	app := cli.App("backoops", "Perform backups")

	swiftURL := app.String(cli.StringOpt{
		Name:   "swift-auth-url",
		Value:  "",
		Desc:   "Swift auth URL",
		EnvVar: "OS_AUTH_URL",
	})
	swiftUser := app.String(cli.StringOpt{
		Name:   "swift-user",
		Value:  "",
		Desc:   "Swift username",
		EnvVar: "OS_USERNAME",
	})
	swiftAPIKey := app.String(cli.StringOpt{
		Name:   "swift-password",
		Value:  "",
		Desc:   "Swift API Key / Password",
		EnvVar: "OS_PASSWORD",
	})
	swiftTenantName := app.String(cli.StringOpt{
		Name:   "swift-tenant-name",
		Value:  "",
		Desc:   "Swift Tenant name",
		EnvVar: "OS_TENANT_NAME",
	})

	app.Command("daemon", "Start the backup process", func(cmd *cli.Cmd) {

		cmd.Spec = "-w... [--etcd]"

		etcdEndpoints := getEtcdOptionsFromCli(cmd)

		watchDirs := cmd.StringsOpt("w watch", []string{}, "Specifies the directories to watch for finding backup.yml files")

		cmd.Action = func() {

			ctx, cancel := context.WithCancel(context.Background())

			// prepare options
			opts := options.NewDefaultOptions()
			opts.EtcdEndpoints = strings.Split(*etcdEndpoints, ",")
			opts.WatchDirs = *watchDirs
			opts.Swift = options.SwiftOptions{
				AuthURL:       *swiftURL,
				User:          *swiftUser,
				APIKey:        *swiftAPIKey,
				TenantName:    *swiftTenantName,
				ContainerName: swiftContainerName,
			}

			ctx = options.NewContext(ctx, opts)

			// handle the SIGINT signal
			waiting := make(chan os.Signal, 1)
			signal.Notify(waiting, os.Interrupt)

			// store the state of the running backups (by project)
			runningState := make(chan map[string]bool)

			// start backup fetching daemon
			go services.FetchBackupConfig(ctx, runningState)
			// start backup routine
			go services.PerformBackup(ctx, runningState)

			// waiting for signal
			<-waiting

			// cancelling ctx
			cancel()

			// check https://blog.golang.org/pipelines
			// for {
			// 	select {
			// 	case <-waiting:
			// 		cancel()
			// 	case <-ctx.Done():
			// 		fmt.Println("\n Exiting.")
			// 		return
			// 	}

			// }
		}
	})

	app.Command("get", "List the available backup archives for a project", func(cmd *cli.Cmd) {

		cmd.Spec = "NAME"

		name := cmd.StringArg("NAME", "", "The name of the project")

		cmd.Action = func() {

			fmt.Println("Searching...")

			swiftOptions := options.SwiftOptions{
				AuthURL:       *swiftURL,
				User:          *swiftUser,
{'fraction_non_alphanumeric': 0.14991624790619765, 'fraction_numerical': 0.00033500837520938025, 'mean_word_length': 3.1179310344827584, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3117330', 'n_tokens_mistral': 2433, 'n_tokens_neox': 2109, 'n_words': 468}