<commit_before>// Copyright 2018 The Go Cloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package s3blob provides a blob implementation that uses S3. Use OpenBucket
// to construct a *blob.Bucket.
//
// Open URLs
//
// For blob.Open URLs, s3blob registers for the scheme "s3"; URLs start
// with "s3://".
//
// The URL's Host is used as the bucket name.
// The AWS session is created as described in
// https://docs.aws.amazon.com/sdk-for-go/api/aws/session/.
// The following query options are supported:
//  - region: The AWS region for requests; sets aws.Config.Region.
// Example URL:
//  s3://mybucket?region=us-east-1
//
// As
//
// s3blob exposes the following types for As:
//  - Bucket: *s3.S3
//  - Error: awserr.Error
//  - ListObject: s3.Object for objects, s3.CommonPrefix for "directories"
//  - ListOptions.BeforeList: *s3.ListObjectsV2Input
//  - Reader: s3.GetObjectOutput
//  - Attributes: s3.HeadObjectOutput
//  - WriterOptions.BeforeWrite: *s3manager.UploadInput
package s3blob // import "gocloud.dev/blob/s3blob"

import (
	"context"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"

	"gocloud.dev/blob"
	"gocloud.dev/blob/driver"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/client"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

const defaultPageSize = 1000

func init() {
	blob.Register("s3", openURL)
}

func openURL(ctx context.Context, u *url.URL) (driver.Bucket, error) {
	q := u.Query()
	cfg := &aws.Config{}

	if region := q["region"]; len(region) > 0 {
		cfg.Region = aws.String(region[0])
	}
	sess, err := session.NewSession(cfg)<commit_msg>blob/s3blob: add endpoint, disableSSL, and s3ForcePathStyle query options (#1095)<commit_after>// Copyright 2018 The Go Cloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package s3blob provides a blob implementation that uses S3. Use OpenBucket
// to construct a *blob.Bucket.
//
// Open URLs
//
// For blob.Open URLs, s3blob registers for the scheme "s3"; URLs start
// with "s3://".
//
// The URL's Host is used as the bucket name.
// The AWS session is created as described in
// https://docs.aws.amazon.com/sdk-for-go/api/aws/session/.
// The following query options are supported:
//  - region: The AWS region for requests; sets aws.Config.Region.
//  - endpoint: The endpoint URL (hostname only or fully qualified URI); sets aws.Config.Endpoint.
//  - disableSSL: A value of "true" disables SSL when sending requests; sets aws.Config.DisableSSL.
//  - s3ForcePathStyle: A value of "true" forces the request to use path-style addressing; sets aws.Config.S3ForcePathStyle.
// Example URL:
//  s3://mybucket?region=us-east-1
//
// As
//
// s3blob exposes the following types for As:
//  - Bucket: *s3.S3
//  - Error: awserr.Error
//  - ListObject: s3.Object for objects, s3.CommonPrefix for "directories"
//  - ListOptions.BeforeList: *s3.ListObjectsV2Input
//  - Reader: s3.GetObjectOutput
//  - Attributes: s3.HeadObjectOutput
//  - WriterOptions.BeforeWrite: *s3manager.UploadInput
package s3blob // import "gocloud.dev/blob/s3blob"

import (
	"context"
	"encoding/base64"
	"encoding/hex"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"

	"gocloud.dev/blob"
	"gocloud.dev/blob/driver"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/client"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

const defaultPageSize = 1000

func init() {
	blob.Register("s3", openURL)
}

func openURL(ctx context.Context, u *url.URL) (driver.Bucket, error) {
	q := u.Query()
	cfg := &aws.Config{}

	if region := q["region"]; len(region) > 0 {
		cfg.Region = aws.String(region[0])
	}
	if endpoint := q["endpoint"]; len(endpoint) > 0 {
		cfg.Endpoint = aws.String(endpoint[0])
	}
	if disableSSL := q["disableSSL"]; len(disableSSL) > 0 {
		cfg.DisableSSL = aws.Bool(disableSSL[0] == "true")
	}
	if s3ForcePathStyle := q["s3ForcePathStyle"]; len(s3ForcePathStyle) > 0 {
		cfg.S3ForcePathStyle = aws.Bool(s3ForcePathStyle[0] == "true")
	}
	sess, err := session.NewSession(cfg)
{'fraction_non_alphanumeric': 0.14520037278657968, 'fraction_numerical': 0.018080149114631874, 'mean_word_length': 3.434710743801653, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4187362', 'n_tokens_mistral': 1882, 'n_tokens_neox': 1831, 'n_words': 526}