<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"mime"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/kelseyhightower/envconfig"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/cloud"
	"google.golang.org/cloud/storage"
)

// Config is the envconfig compatible struct to store config values that are input to gcsup
type Config struct {
	JWTFileLocation string `required:"true" envconfig:"jwt_file_location"`
	ProjectName     string `required:"true" envconfig:"project_name"`
	BucketName      string `required:"true" envconfig:"bucket_name"`
	LocalFolder     string `required:"true" envconfig:"local_folder"`
}

// FilePath represents a file to upload to GCS
type FilePath struct {
	RelativePath string
	Name         string
}

func main() {
	var conf Config
	if err := envconfig.Process("gcsup", &conf); err != nil {
		fmt.Printf("config error [%s]", err)
		os.Exit(1)
	}

	data, err := ioutil.ReadFile(conf.JWTFileLocation)
	if err != nil {
		fmt.Printf("error reading file [%s]\n", err)
		os.Exit(1)
	}

	jwtConf, err := google.JWTConfigFromJSON(data, storage.ScopeFullControl)
	if err != nil {
		fmt.Printf("error creating JWT config [%s]\n", err)
		os.Exit(1)
	}

	fmt.Printf("Uploading %s to gcs://%s/%s\n", conf.LocalFolder, conf.ProjectName, conf.BucketName)

	ctx := cloud.NewContext(conf.ProjectName, jwtConf.Client(oauth2.NoContext))
	var files []FilePath
	if err := filepath.Walk(conf.LocalFolder, func(path string, fInfo os.FileInfo, err error) error {
		if fInfo.IsDir() {
			return nil
		}
		relPath := strings.TrimPrefix(path, conf.LocalFolder+"/")
		files = append(files, FilePath{RelativePath: relPath, Name: fInfo.Name()})
		return nil
	}); err != nil {
		fmt.Printf("error gathering all files [%s]", err)
		os.Exit(1)
	}
	var wg sync.WaitGroup
	for _, file := range files {
		from := file.RelativePath
		to := strings.TrimPrefix(file.RelativePath, conf.LocalFolder)
		fmt.Printf("uploading %s to %s\n", from, to)
		wg.Add(1)
		go func(ctx context.Context, conf Config, from, to string) {
			defer wg.Done()
			if err := upload(ctx, conf, from, to); err != nil {
				fmt.Printf("ERROR uploading %s to %s (%s)\n", from, to, err)
			}
		}(ctx, conf, from, to)
	}
	fmt.Println("waiting for all uploads to finish...")
	wg.Wait()
	fmt.Println("done")
}

func upload(ctx context.Context, conf Config, from, to string) error {
	w := storage.NewWriter(ctx, conf.BucketName, to)
	defer w.Close()
	w.ACL = []storage.ACLRule{
		storage.ACLRule{Entity: storage.AllUsers, Role: storage.RoleReader},
	}
	extension := from[strings.LastIndex(from, "."):]
	w.ContentType = mime.TypeByExtension(extension)

	fileBytes, err := ioutil.ReadFile(from)
	if err != nil {
		return err
	}
	if _, err := w.Write(fileBytes); err != nil {
		return err
	}
	return nil
}<commit_msg>print error on close<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"mime"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/kelseyhightower/envconfig"
	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/cloud"
	"google.golang.org/cloud/storage"
)

// Config is the envconfig compatible struct to store config values that are input to gcsup
type Config struct {
	JWTFileLocation string `required:"true" envconfig:"jwt_file_location"`
	ProjectName     string `required:"true" envconfig:"project_name"`
	BucketName      string `required:"true" envconfig:"bucket_name"`
	LocalFolder     string `required:"true" envconfig:"local_folder"`
}

// FilePath represents a file to upload to GCS
type FilePath struct {
	RelativePath string
	Name         string
}

func main() {
	var conf Config
	if err := envconfig.Process("gcsup", &conf); err != nil {
		fmt.Printf("config error [%s]", err)
		os.Exit(1)
	}

	data, err := ioutil.ReadFile(conf.JWTFileLocation)
	if err != nil {
		fmt.Printf("error reading file [%s]\n", err)
		os.Exit(1)
	}

	jwtConf, err := google.JWTConfigFromJSON(data, storage.ScopeFullControl)
	if err != nil {
		fmt.Printf("error creating JWT config [%s]\n", err)
		os.Exit(1)
	}

	fmt.Printf("Uploading %s to gcs://%s/%s\n", conf.LocalFolder, conf.ProjectName, conf.BucketName)

	ctx := cloud.NewContext(conf.ProjectName, jwtConf.Client(oauth2.NoContext))
	var files []FilePath
	if err := filepath.Walk(conf.LocalFolder, func(path string, fInfo os.FileInfo, err error) error {
		if fInfo.IsDir() {
			return nil
		}
		relPath := strings.TrimPrefix(path, conf.LocalFolder+"/")
		files = append(files, FilePath{RelativePath: relPath, Name: fInfo.Name()})
		return nil
	}); err != nil {
		fmt.Printf("error gathering all files [%s]", err)
		os.Exit(1)
	}
	var wg sync.WaitGroup
	for _, file := range files {
		from := file.RelativePath
		to := strings.TrimPrefix(file.RelativePath, conf.LocalFolder)
		fmt.Printf("uploading %s to %s\n", from, to)
		wg.Add(1)
		go func(ctx context.Context, conf Config, from, to string) {
			defer wg.Done()
			if err := upload(ctx, conf, from, to); err != nil {
				fmt.Printf("ERROR uploading %s to %s (%s)\n", from, to, err)
			}
		}(ctx, conf, from, to)
	}
	fmt.Println("waiting for all uploads to finish...")
	wg.Wait()
	fmt.Println("done")
}

func upload(ctx context.Context, conf Config, from, to string) error {
	w := storage.NewWriter(ctx, conf.BucketName, to)

	defer func() {
		if err := w.Close(); err != nil {
			fmt.Printf("ERROR closing writer for upload %s => %s (%s)\n", from, to, err)
		}
	}()

	w.ACL = []storage.ACLRule{
		storage.ACLRule{Entity: storage.AllUsers, Role: storage.RoleReader},
	}
	extension := from[strings.LastIndex(from, "."):]
	w.ContentType = mime.TypeByExtension(extension)

	fileBytes, err := ioutil.ReadFile(from)
	if err != nil {
		return err
	}
	if _, err := w.Write(fileBytes); err != nil {
		return err
	}
	return nil
}
{'fraction_non_alphanumeric': 0.15121279889901945, 'fraction_numerical': 0.0027524514020299327, 'mean_word_length': 4.077729257641922, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1602934', 'n_tokens_mistral': 2204, 'n_tokens_neox': 2097, 'n_words': 527}