<commit_before>package main

import (
	"archive/zip"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/crowdmob/goamz/aws"
	"github.com/kr/s3"
	"github.com/kr/s3/s3util"
	"github.com/sqs/go-elasticbeanstalk/elasticbeanstalk"
)

var verbose = flag.Bool("v", false, "show verbose output")

var elasticbeanstalkURL *url.URL

func initEnv() {
	var err error
	elasticbeanstalkURL, err = url.Parse(os.Getenv("ELASTICBEANSTALK_URL"))
	if err != nil {
		log.Fatal("Parsing ELASTICBEANSTALK_URL:", err)
	}
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: ebc command [OPTS] ARGS...\n")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "The commands are:")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "\tbundle FILE...\t uploads a source bundle containing the specified files")
		fmt.Fprintln(os.Stderr)
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Environment variables:")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "\tAWS_ACCESS_KEY_ID")
		fmt.Fprintln(os.Stderr, "\tAWS_SECRET_ACCESS_KEY")
		fmt.Fprintln(os.Stderr, "\tELASTICBEANSTALK_URL (default: https://elasticbeanstalk.us-east-1.amazonaws.com)")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Run `ebc command -h` for more information.")
		os.Exit(1)
	}

	flag.Parse()
	initEnv()

	if flag.NArg() == 0 {
		flag.Usage()
	}
	log.SetFlags(0)

	subcmd := flag.Arg(0)
	remaining := flag.Args()[1:]
	switch subcmd {
	case "bundle":
		bundleCmd(remaining)
	}
}

func bundleCmd(args []string) {
	fs := flag.NewFlagSet("bundle", flag.ExitOnError)
	env := fs.String("env", "", "EB environment name")
	app := fs.String("app", "", "EB application name")
	bucket := fs.String("bucket", "", "S3 bucket URL (example: https://example-bucket.s3-us-east-1.amazonaws.com)")
	label := fs.String("label", "", "label base name (suffix of -0, -1, -2, etc., is appended to ensure uniqueness)")
	fs.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: ebc bundle [OPTS] FILE...\n")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Uploads a source bundle containing the specified files.")
		fmt.Fprintln(os.Stderr)
		fs.PrintDefaults()
		fmt.Fprintln(os.Stderr)
		os.Exit(1)
	}
	fs.Parse(args)

	if *env == "" {
		fmt.Fprintln(os.Stderr, "env is required")
		fs.Usage()
	}

	if *app == "" {
		fmt.Fprintln(os.Stderr, "app is required")
		fs.Usage()
	}

	if *bucket == "" {
		fmt.Fprintln(os.Stderr, "bucket is required")
		fs.Usage()
	}
	bucketURL, err := url.Parse(*bucket)
	if err != nil {
		log.Fatal("parsing bucket URL:", err)
	}

	if *label == "" {
		fmt.Fprintln(os.Stderr, "label is required")
		fs.Usage()
	}

	err = bundle(fs.Args(), *env, *app, bucketURL, *label)
	if err != nil {
		log.Fatal("bundle:", err)
	}
}

var s3Config = s3util.Config{
	Keys: &s3.Keys{
		AccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
		SecretKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
	},
	Service: s3.DefaultService,
	Client:  http.DefaultClient,
}

func bundle(paths []string, env, app string, bucketURL *url.URL, label string) error {
	u, fullLabel, err := makeBundleObjectURL(bucketURL, label)
	if err != nil {
		return err
	}

	var t0 time.Time
	if *verbose {
		t0 = time.Now()
		log.Printf("Uploading source bundle to %s...", u.String())
	}
	w, err := s3util.Create(u.String(), nil, &s3Config)
	if err != nil {
		return err
	}<commit_msg>bundle -> deploy<commit_after>package main

import (
	"archive/zip"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	"github.com/crowdmob/goamz/aws"
	"github.com/kr/s3"
	"github.com/kr/s3/s3util"
	"github.com/sqs/go-elasticbeanstalk/elasticbeanstalk"
)

var verbose = flag.Bool("v", false, "show verbose output")

var elasticbeanstalkURL *url.URL

func initEnv() {
	var err error
	elasticbeanstalkURL, err = url.Parse(os.Getenv("ELASTICBEANSTALK_URL"))
	if err != nil {
		log.Fatal("Parsing ELASTICBEANSTALK_URL:", err)
	}
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: ebc command [OPTS] ARGS...\n")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "The commands are:")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "\tdeploy FILE...\t uploads and deploys a source bundle containing the specified files")
		fmt.Fprintln(os.Stderr)
		flag.PrintDefaults()
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Environment variables:")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "\tAWS_ACCESS_KEY_ID")
		fmt.Fprintln(os.Stderr, "\tAWS_SECRET_ACCESS_KEY")
		fmt.Fprintln(os.Stderr, "\tELASTICBEANSTALK_URL (default: https://elasticbeanstalk.us-east-1.amazonaws.com)")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Run `ebc command -h` for more information.")
		os.Exit(1)
	}

	flag.Parse()
	initEnv()

	if flag.NArg() == 0 {
		flag.Usage()
	}
	log.SetFlags(0)

	subcmd := flag.Arg(0)
	remaining := flag.Args()[1:]
	switch subcmd {
	case "deploy":
		deployCmd(remaining)
	}
}

func deployCmd(args []string) {
	fs := flag.NewFlagSet("deploy", flag.ExitOnError)
	env := fs.String("env", "", "EB environment name")
	app := fs.String("app", "", "EB application name")
	bucket := fs.String("bucket", "", "S3 bucket URL (example: https://example-bucket.s3-us-east-1.amazonaws.com)")
	label := fs.String("label", "", "label base name (suffix of -0, -1, -2, etc., is appended to ensure uniqueness)")
	fs.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: ebc deploy [OPTS] FILE...\n")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Uploads and deploys a source bundle containing the specified files.")
		fmt.Fprintln(os.Stderr)
		fs.PrintDefaults()
		fmt.Fprintln(os.Stderr)
		os.Exit(1)
	}
	fs.Parse(args)

	if *env == "" {
		fmt.Fprintln(os.Stderr, "env is required")
		fs.Usage()
	}

	if *app == "" {
		fmt.Fprintln(os.Stderr, "app is required")
		fs.Usage()
	}

	if *bucket == "" {
		fmt.Fprintln(os.Stderr, "bucket is required")
		fs.Usage()
	}
	bucketURL, err := url.Parse(*bucket)
	if err != nil {
		log.Fatal("parsing bucket URL:", err)
	}

	if *label == "" {
		fmt.Fprintln(os.Stderr, "label is required")
		fs.Usage()
	}

	err = deploy(fs.Args(), *env, *app, bucketURL, *label)
	if err != nil {
		log.Fatal("deploy:", err)
	}
}

var s3Config = s3util.Config{
	Keys: &s3.Keys{
		AccessKey: os.Getenv("AWS_ACCESS_KEY_ID"),
		SecretKey: os.Getenv("AWS_SECRET_ACCESS_KEY"),
	},
	Service: s3.DefaultService,
	Client:  http.DefaultClient,
}

func deploy(paths []string, env, app string, bucketURL *url.URL, label string) error {
	u, fullLabel, err := makeBundleObjectURL(bucketURL, label)
	if err != nil {
		return err
	}

	var t0 time.Time
	if *verbose {
		t0 = time.Now()
		log.Printf("Uploading source bundle to %s...", u.String())
	}
	w, err := s3util.Create(u.String(), nil, &s3Config)
	if err != nil {
		return err
	}
{'fraction_non_alphanumeric': 0.17577266735022704, 'fraction_numerical': 0.007030906694009082, 'mean_word_length': 4.020588235294118, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '294121', 'n_tokens_mistral': 2882, 'n_tokens_neox': 2713, 'n_words': 563}