<commit_before>/*
 * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	. "gopkg.in/check.v1"
)

// API suite container common to both FS and XL.
type TestSuiteCommon struct {
	serverType string
	testServer TestServer
	endPoint   string
	accessKey  string
	secretKey  string
}

// Init and run test on FS backend.
var _ = Suite(&TestSuiteCommon{serverType: "FS"})

// Init and run test on XL backend.
var _ = Suite(&TestSuiteCommon{serverType: "XL"})

// Setting up the test suite.
// Starting the Test server with temporary FS backend.
func (s *TestSuiteCommon) SetUpSuite(c *C) {
	s.testServer = StartTestServer(c, s.serverType)
	s.endPoint = s.testServer.Server.URL
	s.accessKey = s.testServer.AccessKey
	s.secretKey = s.testServer.SecretKey
}

// Called implicitly by "gopkg.in/check.v1" after all tests are run.
func (s *TestSuiteCommon) TearDownSuite(c *C) {
	s.testServer.Stop()
}

func (s *TestSuiteCommon) TestAuth(c *C) {
	secretID, err := genSecretAccessKey()
	c.Assert(err, IsNil)

	accessID, err := genAccessKeyID()
	c.Assert(err, IsNil)

	c.Assert(len(secretID), Equals, minioSecretID)
	c.Assert(len(accessID), Equals, minioAccessID)
}

// TestBucketNotification - Inserts the bucket notification and verifies it by fetching the notification back.
func (s *TestSuiteCommon) TestBucketNotification(c *C) {
	// Sample bucket notification.
	bucketNotificationBuf := `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:listen</Topic></TopicConfiguration></NotificationConfiguration>`

	// generate a random bucket Name.
	bucketName := getRandomBucketName()
	// HTTP request to create the bucket.
	request, err := newTestSignedRequest("PUT", getMakeBucketURL(s.endPoint, bucketName),
		0, nil, s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client := http.Client{}
	// execute the request.
	response, err := client.Do(request)
	c.Assert(err, IsNil)
	// assert the http response status code.
	c.Assert(response.StatusCode, Equals, http.StatusOK)

	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(bucketNotificationBuf)), bytes.NewReader([]byte(bucketNotificationBuf)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	// execute the HTTP request.
	response, err = client.Do(request)

	c.Assert(err, IsNil)
	c.Assert(response.StatusCode, Equals, http.StatusOK)

	// Fetch the uploaded policy.
	request, err = newTestSignedRequest("GET", getGetNotificationURL(s.endPoint, bucketName), 0, nil,
		s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	response, err = client.Do(request)
	c.Assert(err, IsNil)
	c.Assert(response.StatusCode, Equals, http.StatusOK)

	bucketNotificationReadBuf, err := ioutil.ReadAll(response.Body)
	c.Assert(err, IsNil)
	// Verify if downloaded policy matches with previousy uploaded.
	c.Assert(bytes.Equal([]byte(bucketNotificationBuf), bucketNotificationReadBuf), Equals, true)

	invalidBucketNotificationBuf := `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:minio</Topic></TopicConfiguration></NotificationConfiguration>`

	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(invalidBucketNotificationBuf)), bytes.NewReader([]byte(invalidBucketNotificationBuf)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	// execute the HTTP request.
	response, err = client.Do(request)
	c.Assert(err, IsNil)

	verifyError(c, response, "InvalidArgument", "A specified destination ARN does not exist or is not well-formed. Verify the destination ARN.", http.StatusBadRequest)

	invalidBucketNotificationBuf = `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-west-1:444455556666:listen</Topic></TopicConfiguration></NotificationConfiguration>`
	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(invalidBucketNotificationBuf)), bytes.NewReader([]byte(invalidBucketNotificationBuf)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	// execute the HTTP request.
	response, err = client.Do(request)
	c.Assert(err, IsNil)

	verifyError(c, response, "InvalidArgument", "A specified destination is in a different region than the bucket. You must use a destination that resides in the same region as the bucket.", http.StatusBadRequest)

	invalidBucketNotificationBuf = `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Invalid</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:listen</Topic></TopicConfiguration></NotificationConfiguration>`
	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(invalidBucketNotificationBuf)), bytes.NewReader([]byte(invalidBucketNotificationBuf)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	// execute the HTTP request.
	response, err = client.Do(request)
	c.Assert(err, IsNil)
	verifyError(c, response, "InvalidArgument", "A specified event is not supported for notifications.", http.StatusBadRequest)

	bucketNotificationDuplicates := `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:listen</Topic></TopicConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:listen</Topic></TopicConfiguration></NotificationConfiguration>`
	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(bucketNotificationDuplicates)), bytes.NewReader([]byte(bucketNotificationDuplicates)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)<commit_msg>tests: Add tests for bucket-notification-utils (#2726)<commit_after>/*
 * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	. "gopkg.in/check.v1"
)

// API suite container common to both FS and XL.
type TestSuiteCommon struct {
	serverType string
	testServer TestServer
	endPoint   string
	accessKey  string
	secretKey  string
}

// Init and run test on FS backend.
var _ = Suite(&TestSuiteCommon{serverType: "FS"})

// Init and run test on XL backend.
var _ = Suite(&TestSuiteCommon{serverType: "XL"})

// Setting up the test suite.
// Starting the Test server with temporary FS backend.
func (s *TestSuiteCommon) SetUpSuite(c *C) {
	s.testServer = StartTestServer(c, s.serverType)
	s.endPoint = s.testServer.Server.URL
	s.accessKey = s.testServer.AccessKey
	s.secretKey = s.testServer.SecretKey
}

// Called implicitly by "gopkg.in/check.v1" after all tests are run.
func (s *TestSuiteCommon) TearDownSuite(c *C) {
	s.testServer.Stop()
}

func (s *TestSuiteCommon) TestAuth(c *C) {
	secretID, err := genSecretAccessKey()
	c.Assert(err, IsNil)

	accessID, err := genAccessKeyID()
	c.Assert(err, IsNil)

	c.Assert(len(secretID), Equals, minioSecretID)
	c.Assert(len(accessID), Equals, minioAccessID)
}

func (s *TestSuiteCommon) TestBucketSQSNotification(c *C) {
	// Sample bucket notification.
	bucketNotificationBuf := `<NotificationConfiguration><QueueConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Queue>arn:minio:sqs:us-east-1:444455556666:amqp</Queue></QueueConfiguration></NotificationConfiguration>`
	// generate a random bucket Name.
	bucketName := getRandomBucketName()
	// HTTP request to create the bucket.
	request, err := newTestSignedRequest("PUT", getMakeBucketURL(s.endPoint, bucketName),
		0, nil, s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client := http.Client{}
	// execute the request.
	response, err := client.Do(request)
	c.Assert(err, IsNil)

	// assert the http response status code.
	c.Assert(response.StatusCode, Equals, http.StatusOK)

	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(bucketNotificationBuf)), bytes.NewReader([]byte(bucketNotificationBuf)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	// execute the HTTP request.
	response, err = client.Do(request)

	c.Assert(err, IsNil)
	verifyError(c, response, "InvalidArgument", "A specified destination ARN does not exist or is not well-formed. Verify the destination ARN.", http.StatusBadRequest)
}

// TestBucketNotification - Inserts the bucket notification and verifies it by fetching the notification back.
func (s *TestSuiteCommon) TestBucketSNSNotification(c *C) {
	// Sample bucket notification.
	bucketNotificationBuf := `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:listen</Topic></TopicConfiguration></NotificationConfiguration>`

	// generate a random bucket Name.
	bucketName := getRandomBucketName()
	// HTTP request to create the bucket.
	request, err := newTestSignedRequest("PUT", getMakeBucketURL(s.endPoint, bucketName),
		0, nil, s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client := http.Client{}
	// execute the request.
	response, err := client.Do(request)
	c.Assert(err, IsNil)
	// assert the http response status code.
	c.Assert(response.StatusCode, Equals, http.StatusOK)

	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(bucketNotificationBuf)), bytes.NewReader([]byte(bucketNotificationBuf)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	// execute the HTTP request.
	response, err = client.Do(request)

	c.Assert(err, IsNil)
	c.Assert(response.StatusCode, Equals, http.StatusOK)

	// Fetch the uploaded policy.
	request, err = newTestSignedRequest("GET", getGetNotificationURL(s.endPoint, bucketName), 0, nil,
		s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	response, err = client.Do(request)
	c.Assert(err, IsNil)
	c.Assert(response.StatusCode, Equals, http.StatusOK)

	bucketNotificationReadBuf, err := ioutil.ReadAll(response.Body)
	c.Assert(err, IsNil)
	// Verify if downloaded policy matches with previousy uploaded.
	c.Assert(bytes.Equal([]byte(bucketNotificationBuf), bucketNotificationReadBuf), Equals, true)

	invalidBucketNotificationBuf := `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>invalid</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:minio</Topic></TopicConfiguration></NotificationConfiguration>`

	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(invalidBucketNotificationBuf)), bytes.NewReader([]byte(invalidBucketNotificationBuf)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	// execute the HTTP request.
	response, err = client.Do(request)
	c.Assert(err, IsNil)

	verifyError(c, response, "InvalidArgument", "A specified destination ARN does not exist or is not well-formed. Verify the destination ARN.", http.StatusBadRequest)

	invalidBucketNotificationBuf = `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>invalid</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:1:listen</Topic></TopicConfiguration></NotificationConfiguration>`

	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(invalidBucketNotificationBuf)), bytes.NewReader([]byte(invalidBucketNotificationBuf)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	// execute the HTTP request.
	response, err = client.Do(request)
	c.Assert(err, IsNil)

	verifyError(c, response, "InvalidArgument", "filter rule name must be either prefix or suffix", http.StatusBadRequest)

	invalidBucketNotificationBuf = `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>|||</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:1:listen</Topic></TopicConfiguration></NotificationConfiguration>`

	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(invalidBucketNotificationBuf)), bytes.NewReader([]byte(invalidBucketNotificationBuf)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	// execute the HTTP request.
	response, err = client.Do(request)
	c.Assert(err, IsNil)

	verifyError(c, response, "InvalidArgument", "Size of filter rule value cannot exceed 1024 bytes in UTF-8 representation", http.StatusBadRequest)

	invalidBucketNotificationBuf = `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-west-1:444455556666:listen</Topic></TopicConfiguration></NotificationConfiguration>`
	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(invalidBucketNotificationBuf)), bytes.NewReader([]byte(invalidBucketNotificationBuf)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	// execute the HTTP request.
	response, err = client.Do(request)
	c.Assert(err, IsNil)

	verifyError(c, response, "InvalidArgument", "A specified destination is in a different region than the bucket. You must use a destination that resides in the same region as the bucket.", http.StatusBadRequest)

	invalidBucketNotificationBuf = `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Invalid</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:listen</Topic></TopicConfiguration></NotificationConfiguration>`
	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(invalidBucketNotificationBuf)), bytes.NewReader([]byte(invalidBucketNotificationBuf)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)

	client = http.Client{}
	// execute the HTTP request.
	response, err = client.Do(request)
	c.Assert(err, IsNil)
	verifyError(c, response, "InvalidArgument", "A specified event is not supported for notifications.", http.StatusBadRequest)

	bucketNotificationDuplicates := `<NotificationConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:listen</Topic></TopicConfiguration><TopicConfiguration><Event>s3:ObjectCreated:Put</Event><Filter><S3Key><FilterRule><Name>prefix</Name><Value>images/</Value></FilterRule></S3Key></Filter><Id>1</Id><Topic>arn:minio:sns:us-east-1:444455556666:listen</Topic></TopicConfiguration></NotificationConfiguration>`
	request, err = newTestSignedRequest("PUT", getPutNotificationURL(s.endPoint, bucketName),
		int64(len(bucketNotificationDuplicates)), bytes.NewReader([]byte(bucketNotificationDuplicates)), s.accessKey, s.secretKey)
	c.Assert(err, IsNil)
{'fraction_non_alphanumeric': 0.1404090026276705, 'fraction_numerical': 0.017536844510453557, 'mean_word_length': 6.546120689655172, 'pattern_counts': {'":': 0, '<': 299, '<?xml version=': 0, '>': 299, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3053207', 'n_tokens_mistral': 5625, 'n_tokens_neox': 5370, 'n_words': 1152}