<commit_before>package main_test

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"strings"
	"time"

	yaml "gopkg.in/yaml.v2"

	"strconv"

	"github.com/cloudfoundry-incubator/bits-service/blobstores/alibaba"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/azure"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/gcp"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/openstack"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/s3"
	"github.com/cloudfoundry-incubator/bits-service/config"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Non-local blobstores SLOW TESTS", func() {

	BeforeSuite(func() {
		rand.Seed(time.Now().Unix())
	})

	var (
		filepath  string
		blobstore blobstore
	)

	slowTests := func() {

		It("can upload a 500MB file", func() {
			filename, expectedMd5sum := createTempFileWithRandomContent(500 << 20)
			defer os.Remove(filename)

			file, e := os.Open(filename)
			Expect(e).NotTo(HaveOccurred())
			defer file.Close()

			e = blobstore.Put(filepath, file)
			Expect(e).NotTo(HaveOccurred())
			defer blobstore.Delete(filepath)

			reader, e := blobstore.Get(filepath)
			Expect(e).NotTo(HaveOccurred())
			defer reader.Close()

			Expect(md5sum(reader)).To(Equal(expectedMd5sum))
		})

		It("Can delete a dir with many files in it", func() {
			numManyFiles := 10000
			dirname := strconv.Itoa(rand.Int())
			randomFilenames := generateRandomFilenames(numManyFiles, dirname)

			By("Uploading files...")
			uploadFiles(blobstore, randomFilenames)
			By("Files uploaded.")

			By("Deleting dir...")
			Expect(blobstore.DeleteDir(dirname)).To(Succeed())
			By("Dir deleted.")

			By("Checking existence...")
			assertFilesDoNotExist(blobstore, randomFilenames)
			By("Existence checked.")
		})
	}

	var configFileContent []byte

	BeforeEach(func() {
		filename := os.Getenv("CONFIG")
		if filename == "" {
			fmt.Println("No $CONFIG set. Defaulting to integration_test_config.yml")
			filename = "integration_test_config.yml"
		}
		file, e := os.Open(filename)
		Expect(e).NotTo(HaveOccurred())
		defer file.Close()
		configFileContent, e = ioutil.ReadAll(file)
		Expect(e).NotTo(HaveOccurred())

		filepath = fmt.Sprintf("testfile-%v", time.Now())
	})

	Context("S3", func() {
		var s3Config config.S3BlobstoreConfig

		BeforeEach(func() {
			Expect(yaml.Unmarshal(configFileContent, &s3Config)).To(Succeed())

			if strings.Contains(s3Config.Host, "google") {
				Skip("Not running high-traffic tests on GCP yet.")
			}
		})
		JustBeforeEach(func() { blobstore = s3.NewBlobstore(s3Config) })

		slowTests()
	})

	PContext("GCP", func() {
		var gcpConfig config.GCPBlobstoreConfig

		BeforeEach(func() { Expect(yaml.Unmarshal(configFileContent, &gcpConfig)).To(Succeed()) })
		JustBeforeEach(func() { blobstore = gcp.NewBlobstore(gcpConfig) })

		slowTests()
	})

	Context("azure", func() {
		var azureConfig config.AzureBlobstoreConfig

		BeforeEach(func() { Expect(yaml.Unmarshal(configFileContent, &azureConfig)).To(Succeed()) })
		JustBeforeEach(func() { blobstore = azure.NewBlobstore(azureConfig) })
<commit_msg>Include GCP slow contracts integ tests<commit_after>package main_test

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"os"
	"strings"
	"time"

	yaml "gopkg.in/yaml.v2"

	"strconv"

	"github.com/cloudfoundry-incubator/bits-service/blobstores/alibaba"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/azure"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/gcp"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/openstack"
	"github.com/cloudfoundry-incubator/bits-service/blobstores/s3"
	"github.com/cloudfoundry-incubator/bits-service/config"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Non-local blobstores SLOW TESTS", func() {

	BeforeSuite(func() {
		rand.Seed(time.Now().Unix())
	})

	var (
		filepath  string
		blobstore blobstore
	)

	slowTests := func() {

		It("can upload a 500MB file", func() {
			filename, expectedMd5sum := createTempFileWithRandomContent(500 << 20)
			defer os.Remove(filename)

			file, e := os.Open(filename)
			Expect(e).NotTo(HaveOccurred())
			defer file.Close()

			e = blobstore.Put(filepath, file)
			Expect(e).NotTo(HaveOccurred())
			defer blobstore.Delete(filepath)

			reader, e := blobstore.Get(filepath)
			Expect(e).NotTo(HaveOccurred())
			defer reader.Close()

			Expect(md5sum(reader)).To(Equal(expectedMd5sum))
		})

		It("Can delete a dir with many files in it", func() {
			numManyFiles := 10000
			dirname := strconv.Itoa(rand.Int())
			randomFilenames := generateRandomFilenames(numManyFiles, dirname)

			By("Uploading files...")
			uploadFiles(blobstore, randomFilenames)
			By("Files uploaded.")

			By("Deleting dir...")
			Expect(blobstore.DeleteDir(dirname)).To(Succeed())
			By("Dir deleted.")

			By("Checking existence...")
			assertFilesDoNotExist(blobstore, randomFilenames)
			By("Existence checked.")
		})
	}

	var configFileContent []byte

	BeforeEach(func() {
		filename := os.Getenv("CONFIG")
		if filename == "" {
			fmt.Println("No $CONFIG set. Defaulting to integration_test_config.yml")
			filename = "integration_test_config.yml"
		}
		file, e := os.Open(filename)
		Expect(e).NotTo(HaveOccurred())
		defer file.Close()
		configFileContent, e = ioutil.ReadAll(file)
		Expect(e).NotTo(HaveOccurred())

		filepath = fmt.Sprintf("testfile-%v", time.Now())
	})

	Context("S3", func() {
		var s3Config config.S3BlobstoreConfig

		BeforeEach(func() {
			Expect(yaml.Unmarshal(configFileContent, &s3Config)).To(Succeed())

			if strings.Contains(s3Config.Host, "google") {
				Skip("Not running high-traffic tests on GCP yet.")
			}
		})
		JustBeforeEach(func() { blobstore = s3.NewBlobstore(s3Config) })

		slowTests()
	})

	Context("GCP", func() {
		var gcpConfig config.GCPBlobstoreConfig

		BeforeEach(func() { Expect(yaml.Unmarshal(configFileContent, &gcpConfig)).To(Succeed()) })
		JustBeforeEach(func() { blobstore = gcp.NewBlobstore(gcpConfig) })

		slowTests()
	})

	Context("azure", func() {
		var azureConfig config.AzureBlobstoreConfig

		BeforeEach(func() { Expect(yaml.Unmarshal(configFileContent, &azureConfig)).To(Succeed()) })
		JustBeforeEach(func() { blobstore = azure.NewBlobstore(azureConfig) })

{'fraction_non_alphanumeric': 0.15318744053282587, 'fraction_numerical': 0.008246114811290834, 'mean_word_length': 4.6112099644128115, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '247894', 'n_tokens_mistral': 2525, 'n_tokens_neox': 2238, 'n_words': 357}