<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"io"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
    "math"
)

type ObjectInputStream struct {
	Size      int64
	Pos       int64
	FirstByte bool
	StartTs   time.Time
	CurrentTs time.Time
}

func NewObjectInputStream(size int64) (o *ObjectInputStream) {
	return &ObjectInputStream{
		Size:      size,
		Pos:       0,
		FirstByte: true,
	}
}

func (cin *ObjectInputStream) Read(b []byte) (n int, err error) {
	//fmt.Println("Read pos", cin.Pos)
	if cin.Pos >= cin.Size {
		return 0, io.EOF
	}

	if cin.FirstByte {
		cin.FirstByte = false
		cin.StartTs = time.Now()
	}

	var sz int64 = 0
	if int64(len(b)) > (cin.Size - cin.Pos) {
		sz = (cin.Size - cin.Pos)
	} else {
		sz = int64(len(b))
	}

	buffer := make([]byte, sz)
	_, _ = rand.Read(buffer)
	copied := copy(b, buffer)

	cin.Pos += int64(copied)
	cin.CurrentTs = time.Now()
	n = copied
	return
}

func (cin *ObjectInputStream) Seek(offset int64, whence int) (int64, error) {
	//fmt.Println("seek", offset, whence)
	switch whence {
	case io.SeekStart:
		if offset > cin.Size || (cin.Size+offset) < 0 {
			return 0, io.EOF
		} else {
			cin.Pos = offset
			return offset, nil
		}
	case io.SeekCurrent:
		if (cin.Pos+offset) > cin.Size || (cin.Size+offset) < 0 {
			return 0, io.EOF
		} else {
			cin.Pos += offset
			return offset, nil
		}
	case io.SeekEnd:
		if (cin.Size+offset) > cin.Size || (cin.Size+offset) < 0 {
			return 0, io.EOF
		} else {
			cin.Pos += offset
			return offset, nil
		}
	}

	return 0, nil
}

var bucketname = flag.String("bucket", "cloudian-auto-spots", "The bucket to store your object")
var objname = flag.String("objkey", "test/objtest", "The object name (key)")
var objsize = flag.String("size", "10M", "Object size (default 10MB)")
var partsize = flag.String("partsize", "5M", "The size of parts in bytes used min 5MB")
var region = flag.String("region", "us-east-2", "Region name to be used")
var endpoint = flag.String("endpoint", "", "Overwrite the endpoint")
var profile = flag.String("profile", "default", "The profile name")
var pathstyle = flag.Bool("pathstyle", false, "Enforce path style urls")
var nossl = flag.Bool("nossl", false, "Don't use SSL")
var nomd5 = flag.Bool("nomd5", false, "Disable adding ContentMD5 to S3 object put and uploads")
var nosum = flag.Bool("nosum", false, "Disable creating checksums")
var retries = flag.Int("retries", -1, "Set the number of retries default -1 (forever)")
var maxparts = flag.Int("maxparts", 0, "Number of parts to upload")
var maxthreads = flag.Int("maxthreads", 0, "number of threads to use per object default 0 (sdk default)")
var delparts = flag.Bool("delparts", true, "Delete parts on errors")
var help = flag.Bool("h", false, "Print a helpful message.")

func usage() {
	fmt.Println()
	fmt.Println("Use", os.Args[0])
	fmt.Println("\t-bucket     <bucket_name> default 'cloudian-auto-spots'")
	fmt.Println("\t            will create the bucket if not exist")
	fmt.Println("\t-objkey     <objectname> default 'test/objtest'")
	fmt.Println("\t-size       <size> object size default 10MB")
	fmt.Println("\t            B = bytes, K = kilo bytes M = mega bytes")
	fmt.Println("\t            G = giga bytes, T = terra bytes, example 10K")
	fmt.Println("\t-partsize   <chunk_size> The size of parts used in bytes min 5MB")
	fmt.Println("\t            B = bytes, K = kilo bytes M = mega bytes")
	fmt.Println("\t            G = giga bytes, T = terra bytes, example 10K")
	fmt.Println("\t-region     <region_name> The region name that should be used default us-east-2")
	fmt.Println("\t-endpoint   <endpoint> Set the endpoint to be used default is 'default'")
	fmt.Println("\t            use -nossl if the endpoint is not https")
	fmt.Println("\t-profile    <profile> Set the profile name default is 'default'")
	fmt.Println("\t-pathstyle  If set it enforces path style urls")
	fmt.Println("\t-nossl      Don't use SSL")
	fmt.Println("\t-nomd5      Disable adding ContentMD5 to S3 object put and upload")
	fmt.Println("\t-nosum      Disable creating checksums")
	fmt.Println("\t-retries    Set the number of retries default -1 forever")
	fmt.Println("\t-maxparts   <count> Will devide the object size by this number to calculate the part size")
	fmt.Println("\t            will ignore partsize if set")
	fmt.Println("\t-maxthreads <count> number of threads to use per object default 0 (sdk default)")
	fmt.Println("\t-delparts   Delete parts on error, default true")
	fmt.Println()
}

func unitsToBytes(u string) (r int64, err error) {
	if strings.HasSuffix(strings.ToUpper(u), "B") {
		result, err := strconv.ParseInt(strings.TrimSuffix(u, "B"), 10, 64)
		if err == nil {
			return result, nil
		} else {
			return 0, errors.New("Failed to parse number")
		}
	} else if strings.HasSuffix(strings.ToUpper(u), "K") {
		result, err := strconv.ParseInt(strings.TrimSuffix(u, "K"), 10, 64)
		if err == nil {
			return (result * (2 << 9)), nil
		} else {
			return 0, errors.New("Failed to parse number")
		}
	} else if strings.HasSuffix(strings.ToUpper(u), "M") {
		result, err := strconv.ParseInt(strings.TrimSuffix(u, "M"), 10, 64)
		if err == nil {
			return (result * (2 << 19)), nil
		} else {
			return 0, errors.New("Failed to parse number")
		}
	} else if strings.HasSuffix(strings.ToUpper(u), "G") {
		result, err := strconv.ParseInt(strings.TrimSuffix(u, "G"), 10, 64)
		if err == nil {
			return (result * (2 << 29)), nil
		} else {
			return 0, errors.New("Failed to parse number")
		}
	} else if strings.HasSuffix(strings.ToUpper(u), "T") {
		result, err := strconv.ParseInt(strings.TrimSuffix(u, "T"), 10, 64)
		if err == nil {
			return (result * (2 << 39)), nil
		} else {
			return 0, errors.New("Failed to parse number")
		}
	}

	return 0, errors.New("Unknown suffix")
}

func main() {
	flag.Parse()
	if *help {
		usage()
		return
	}

	osize, err := unitsToBytes(*objsize)
	if err != nil {
		exitErrorf("Error parameter -size %v", err)
	}

	psize, err := unitsToBytes(*partsize)
	if err != nil {
		exitErrorf("Error parameter -partsize %v", err)
	}

	if *maxparts > 0 {
        fmt.Println(*maxparts)
        psize = int64(math.Ceil(float64(osize)/float64(*maxparts)))
        fmt.Println(psize)
	}

	config := aws.NewConfig().
		WithCredentials(credentials.NewSharedCredentials("", *profile)).
		WithEndpoint(*endpoint).
		WithRegion(*region).
		WithDisableSSL(*nossl).
		WithMaxRetries(*retries).
		WithDisableComputeChecksums(*nosum).
		WithS3DisableContentMD5Validation(*nomd5).
		WithS3ForcePathStyle(*pathstyle)
	t := time.Now()
	o := NewObjectInputStream(osize)
	bucket := *bucketname
	filename := *objname

	sess, err := session.NewSession(config)
	if err != nil {
		exitErrorf("Unable to create session %v", err)
	}
	// http://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#NewUploader
	uploader := s3manager.NewUploader(sess, func(u *s3manager.Uploader) {
		u.Concurrency = *maxthreads
		u.LeavePartsOnError = *delparts
		if *maxparts > 0 {
			u.MaxUploadParts = *maxparts
			u.PartSize = psize
			fmt.Println("Using part size", u.PartSize)
		} else {
			u.PartSize = psize
		}
	})

	// Upload the file's body to S3 bucket as an object with the key being the
	// same as the filename.
	_, err = uploader.Upload(&s3manager.UploadInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(filename),

		// The file to be uploaded. io.ReadSeeker is preferred as the Uploader
		// will be able to optimize memory when uploading large content. io.Reader
		// is supported, but will require buffering of the reader's bytes for
		// each part.
		Body: o,
	})

	if err != nil {
		exitErrorf("Unable to upload %q to %q, %v", filename, bucket, err)
	}

	ptime := (o.CurrentTs.Sub(o.StartTs).Seconds())
	utime := (time.Now().Sub(o.StartTs).Seconds())
	rate := ((float64(o.Pos) / (2 << 20)) / utime)
	latency := o.StartTs.Sub(t).Seconds() * 1000
	fmt.Println("#bucketname,objectname,objectsize in bytes,latency in ms,ptime in s,uploadtime in s,transferrate MB/s")
	fmt.Printf("%s,%s,%v,%v,%v,%v,%v\n", bucket, filename, *objsize, latency, ptime, utime, rate)
}

func exitErrorf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}
<commit_msg>after go fmt<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	"io"
	"math"
	"math/rand"
	"os"
	"strconv"
	"strings"
	"time"
)

type ObjectInputStream struct {
	Size      int64
	Pos       int64
	FirstByte bool
	StartTs   time.Time
	CurrentTs time.Time
}

func NewObjectInputStream(size int64) (o *ObjectInputStream) {
	return &ObjectInputStream{
		Size:      size,
		Pos:       0,
		FirstByte: true,
	}
}

func (cin *ObjectInputStream) Read(b []byte) (n int, err error) {
	//fmt.Println("Read pos", cin.Pos)
	if cin.Pos >= cin.Size {
		return 0, io.EOF
	}

	if cin.FirstByte {
		cin.FirstByte = false
		cin.StartTs = time.Now()
	}

	var sz int64 = 0
	if int64(len(b)) > (cin.Size - cin.Pos) {
		sz = (cin.Size - cin.Pos)
	} else {
		sz = int64(len(b))
	}

	buffer := make([]byte, sz)
	_, _ = rand.Read(buffer)
	copied := copy(b, buffer)

	cin.Pos += int64(copied)
	cin.CurrentTs = time.Now()
	n = copied
	return
}

func (cin *ObjectInputStream) Seek(offset int64, whence int) (int64, error) {
	//fmt.Println("seek", offset, whence)
	switch whence {
	case io.SeekStart:
		if offset > cin.Size || (cin.Size+offset) < 0 {
			return 0, io.EOF
		} else {
			cin.Pos = offset
			return offset, nil
		}
	case io.SeekCurrent:
		if (cin.Pos+offset) > cin.Size || (cin.Size+offset) < 0 {
			return 0, io.EOF
		} else {
			cin.Pos += offset
			return offset, nil
		}
	case io.SeekEnd:
		if (cin.Size+offset) > cin.Size || (cin.Size+offset) < 0 {
			return 0, io.EOF
		} else {
			cin.Pos += offset
			return offset, nil
		}
	}

	return 0, nil
}

var bucketname = flag.String("bucket", "cloudian-auto-spots", "The bucket to store your object")
var objname = flag.String("objkey", "test/objtest", "The object name (key)")
var objsize = flag.String("size", "10M", "Object size (default 10MB)")
var partsize = flag.String("partsize", "5M", "The size of parts in bytes used min 5MB")
var region = flag.String("region", "us-east-2", "Region name to be used")
var endpoint = flag.String("endpoint", "", "Overwrite the endpoint")
var profile = flag.String("profile", "default", "The profile name")
var pathstyle = flag.Bool("pathstyle", false, "Enforce path style urls")
var nossl = flag.Bool("nossl", false, "Don't use SSL")
var nomd5 = flag.Bool("nomd5", false, "Disable adding ContentMD5 to S3 object put and uploads")
var nosum = flag.Bool("nosum", false, "Disable creating checksums")
var retries = flag.Int("retries", -1, "Set the number of retries default -1 (forever)")
var maxparts = flag.Int("maxparts", 0, "Number of parts to upload")
var maxthreads = flag.Int("maxthreads", 0, "number of threads to use per object default 0 (sdk default)")
var delparts = flag.Bool("delparts", true, "Delete parts on errors")
var help = flag.Bool("h", false, "Print a helpful message.")

func usage() {
	fmt.Println()
	fmt.Println("Use", os.Args[0])
	fmt.Println("\t-bucket     <bucket_name> default 'cloudian-auto-spots'")
	fmt.Println("\t            will create the bucket if not exist")
	fmt.Println("\t-objkey     <objectname> default 'test/objtest'")
	fmt.Println("\t-size       <size> object size default 10MB")
	fmt.Println("\t            B = bytes, K = kilo bytes M = mega bytes")
	fmt.Println("\t            G = giga bytes, T = terra bytes, example 10K")
	fmt.Println("\t-partsize   <chunk_size> The size of parts used in bytes min 5MB")
	fmt.Println("\t            B = bytes, K = kilo bytes M = mega bytes")
	fmt.Println("\t            G = giga bytes, T = terra bytes, example 10K")
	fmt.Println("\t-region     <region_name> The region name that should be used default us-east-2")
	fmt.Println("\t-endpoint   <endpoint> Set the endpoint to be used default is 'default'")
	fmt.Println("\t            use -nossl if the endpoint is not https")
	fmt.Println("\t-profile    <profile> Set the profile name default is 'default'")
	fmt.Println("\t-pathstyle  If set it enforces path style urls")
	fmt.Println("\t-nossl      Don't use SSL")
	fmt.Println("\t-nomd5      Disable adding ContentMD5 to S3 object put and upload")
	fmt.Println("\t-nosum      Disable creating checksums")
	fmt.Println("\t-retries    Set the number of retries default -1 forever")
	fmt.Println("\t-maxparts   <count> Will devide the object size by this number to calculate the part size")
	fmt.Println("\t            will ignore partsize if set")
	fmt.Println("\t-maxthreads <count> number of threads to use per object default 0 (sdk default)")
	fmt.Println("\t-delparts   Delete parts on error, default true")
	fmt.Println()
}

func unitsToBytes(u string) (r int64, err error) {
	if strings.HasSuffix(strings.ToUpper(u), "B") {
		result, err := strconv.ParseInt(strings.TrimSuffix(u, "B"), 10, 64)
		if err == nil {
			return result, nil
		} else {
			return 0, errors.New("Failed to parse number")
		}
	} else if strings.HasSuffix(strings.ToUpper(u), "K") {
		result, err := strconv.ParseInt(strings.TrimSuffix(u, "K"), 10, 64)
		if err == nil {
			return (result * (2 << 9)), nil
		} else {
			return 0, errors.New("Failed to parse number")
		}
	} else if strings.HasSuffix(strings.ToUpper(u), "M") {
		result, err := strconv.ParseInt(strings.TrimSuffix(u, "M"), 10, 64)
		if err == nil {
			return (result * (2 << 19)), nil
		} else {
			return 0, errors.New("Failed to parse number")
		}
	} else if strings.HasSuffix(strings.ToUpper(u), "G") {
		result, err := strconv.ParseInt(strings.TrimSuffix(u, "G"), 10, 64)
		if err == nil {
			return (result * (2 << 29)), nil
		} else {
			return 0, errors.New("Failed to parse number")
		}
	} else if strings.HasSuffix(strings.ToUpper(u), "T") {
		result, err := strconv.ParseInt(strings.TrimSuffix(u, "T"), 10, 64)
		if err == nil {
			return (result * (2 << 39)), nil
		} else {
			return 0, errors.New("Failed to parse number")
		}
	}

	return 0, errors.New("Unknown suffix")
}

func main() {
	flag.Parse()
	if *help {
		usage()
		return
	}

	osize, err := unitsToBytes(*objsize)
	if err != nil {
		exitErrorf("Error parameter -size %v", err)
	}

	psize, err := unitsToBytes(*partsize)
	if err != nil {
		exitErrorf("Error parameter -partsize %v", err)
	}

	if *maxparts > 0 {
		fmt.Println(*maxparts)
		psize = int64(math.Ceil(float64(osize) / float64(*maxparts)))
		fmt.Println(psize)
	}

	config := aws.NewConfig().
		WithCredentials(credentials.NewSharedCredentials("", *profile)).
		WithEndpoint(*endpoint).
		WithRegion(*region).
		WithDisableSSL(*nossl).
		WithMaxRetries(*retries).
		WithDisableComputeChecksums(*nosum).
		WithS3DisableContentMD5Validation(*nomd5).
		WithS3ForcePathStyle(*pathstyle)
	t := time.Now()
	o := NewObjectInputStream(osize)
	bucket := *bucketname
	filename := *objname

	sess, err := session.NewSession(config)
	if err != nil {
		exitErrorf("Unable to create session %v", err)
	}
	// http://docs.aws.amazon.com/sdk-for-go/api/service/s3/s3manager/#NewUploader
	uploader := s3manager.NewUploader(sess, func(u *s3manager.Uploader) {
		u.Concurrency = *maxthreads
		u.LeavePartsOnError = *delparts
		if *maxparts > 0 {
			u.MaxUploadParts = *maxparts
			u.PartSize = psize
			fmt.Println("Using part size", u.PartSize)
		} else {
			u.PartSize = psize
		}
	})

	// Upload the file's body to S3 bucket as an object with the key being the
	// same as the filename.
	_, err = uploader.Upload(&s3manager.UploadInput{
		Bucket: aws.String(bucket),
		Key:    aws.String(filename),

		// The file to be uploaded. io.ReadSeeker is preferred as the Uploader
		// will be able to optimize memory when uploading large content. io.Reader
		// is supported, but will require buffering of the reader's bytes for
		// each part.
		Body: o,
	})

	if err != nil {
		exitErrorf("Unable to upload %q to %q, %v", filename, bucket, err)
	}

	ptime := (o.CurrentTs.Sub(o.StartTs).Seconds())
	utime := (time.Now().Sub(o.StartTs).Seconds())
	rate := ((float64(o.Pos) / (2 << 20)) / utime)
	latency := o.StartTs.Sub(t).Seconds() * 1000
	fmt.Println("#bucketname,objectname,objectsize in bytes,latency in ms,ptime in s,uploadtime in s,transferrate MB/s")
	fmt.Printf("%s,%s,%v,%v,%v,%v,%v\n", bucket, filename, *objsize, latency, ptime, utime, rate)
}

func exitErrorf(msg string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, msg+"\n", args...)
	os.Exit(1)
}

{'fraction_non_alphanumeric': 0.1531746980748535, 'fraction_numerical': 0.015185938060504604, 'mean_word_length': 3.5802300109529024, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3798276', 'n_tokens_mistral': 6399, 'n_tokens_neox': 5925, 'n_words': 1675}