<commit_before>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/rook/rook/pkg/operator/k8sutil"
	"github.com/rook/rook/tests/framework/clients"
	"github.com/rook/rook/tests/framework/installer"
	"github.com/rook/rook/tests/framework/utils"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
)

const (
	rbdPodName        = "test-pod-upgrade"
	operatorContainer = "rook-ceph-operator"
)

// ************************************************
// *** Major scenarios tested by the UpgradeSuite ***
// Setup
// - Initially create a cluster from the previous minor release
// - Upgrade to the current build of Rook to verify functionality after upgrade
// - Test basic usage of block, object, and file after upgrade
// Monitors
// - One mon in the cluster
// ************************************************
func TestCephUpgradeSuite(t *testing.T) {
	if installer.SkipTestSuite(installer.CephTestSuite) {
		t.Skip()
	}

	// Skip the suite if CSI is not supported
	kh, err := utils.CreateK8sHelper(func() *testing.T { return t })
	require.NoError(t, err)
	checkSkipCSITest(t, kh)

	s := new(UpgradeSuite)
	defer func(s *UpgradeSuite) {
		HandlePanics(recover(), s.op, s.T)
	}(s)
	suite.Run(t, s)
}

type UpgradeSuite struct {
	suite.Suite
	helper    *clients.TestClient
	op        *TestCluster
	k8sh      *utils.K8sHelper
	namespace string
}

func (s *UpgradeSuite) SetupSuite() {
	s.namespace = "upgrade-ns"
	mons := 1
	rbdMirrorWorkers := 0
	s.op, s.k8sh = StartTestCluster(s.T,
		upgradeMinimalTestVersion,
		s.namespace,
		"",
		false,
		false,
		"",
		mons,
		rbdMirrorWorkers,
		installer.Version1_1,
		installer.MimicVersion,
		false,
	)
	s.helper = clients.CreateTestClient(s.k8sh, s.op.installer.Manifests)
}

func (s *UpgradeSuite) TearDownSuite() {
	s.op.Teardown()
}

func (s *UpgradeSuite) TestUpgradeToMaster() {
	systemNamespace := installer.SystemNamespace(s.namespace)

	//
	// Create block, object, and file storage before the upgrade
	//
	poolName := "upgradepool"
	storageClassName := "block-upgrade"
	blockName := "block-claim-upgrade"
	logger.Infof("Initializing block before the upgrade")
	setupBlockLite(s.helper, s.k8sh, s.Suite, s.namespace, systemNamespace, poolName, storageClassName, blockName, rbdPodName, s.op.installer.CephVersion)

	createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, storageClassName, rbdPodName, blockName)

	// FIX: We should require block images to be removed. See tracking issue:
	// <INSERT ISSUE>
	requireBlockImagesRemoved := false
	defer blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, rbdPodName, requireBlockImagesRemoved)

	// Create the filesystem, but wait to create the file test client until we upgrade to nautilus
	logger.Infof("Initializing file before the upgrade")
	filesystemName := "upgrade-test-fs"
	activeCount := 1
	createFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, activeCount)

	logger.Infof("Initializing object before the upgrade")
	objectStoreName := "upgraded-object"
	runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.namespace, objectStoreName, 1, false)

	// verify that we're actually running the right pre-upgrade image
	s.verifyOperatorImage(installer.Version1_1)

	message := "my simple message"
	preFilename := "pre-upgrade-file"
	assert.NoError(s.T(), s.k8sh.WriteToPod("", rbdPodName, preFilename, message))
	assert.NoError(s.T(), s.k8sh.ReadFromPod("", rbdPodName, preFilename, message))

	// we will keep appending to this to continue verifying old files through the upgrades
	rbdFilesToRead := []string{preFilename}
	cephfsFilesToRead := []string{}

	// Get some info about the currently deployed OSDs to determine later if they are all updated
	osdDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd")
	require.NoError(s.T(), err)
	osdDeps := osdDepList.Items
	numOSDs := len(osdDeps) // there should be this many upgraded OSDs
	require.NotEqual(s.T(), 0, numOSDs)

	//
	// Upgrade Rook from v1.1 to v1.2
	//
	logger.Infof("*** UPGRADING ROOK FROM v1.1 to v1.2 ***")
	s.gatherLogs(systemNamespace, "_before_1.2_upgrade")
	s.upgradeToV1_2()

	s.verifyOperatorImage(installer.Version1_2)
	s.verifyRookUpgrade(numOSDs)
	logger.Infof("Done with automatic upgrade from v1.1 to v1.2")
	newFile := "post-upgrade-1_1-to-1_2-file"
	s.verifyFilesAfterUpgrade("", newFile, message, rbdFilesToRead, cephfsFilesToRead)
	rbdFilesToRead = append(rbdFilesToRead, newFile)
	logger.Infof("Verified upgrade from v1.1 to v1.2")

	//
	// Upgrade from mimic to nautilus
	//
	logger.Infof("*** UPGRADING CEPH FROM Mimic TO Nautilus ***")
	s.gatherLogs(systemNamespace, "_before_ceph_upgrade")
	s.upgradeCephVersion(installer.NautilusVersion.Image, numOSDs)

	// Start the file test client now that the CSI driver is supported on nautilus
	fsStorageClass := "file-upgrade"
	assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, s.namespace, fsStorageClass))
	useCSI := true
	createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, fsStorageClass, useCSI)
	defer func() {
		cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName)
		cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName)
	}()

	// Verify reading and writing to the test clients
	newFile = "post-ceph-upgrade-file"
	s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead)
	rbdFilesToRead = append(rbdFilesToRead, newFile)
	cephfsFilesToRead = append(cephfsFilesToRead, newFile)
	logger.Infof("Verified upgrade from mimic to nautilus")

	//
	// Upgrade Rook from v1.2 to master
	//
	logger.Infof("*** UPGRADING ROOK FROM v1.2 to master ***")
	s.gatherLogs(systemNamespace, "_before_master_upgrade")
	s.upgradeToMaster()

	s.verifyOperatorImage(installer.VersionMaster)
	s.verifyRookUpgrade(numOSDs)
	logger.Infof("Done with automatic upgrade from v1.2 to master")
	newFile = "post-upgrade-1_2-to-master-file"
	s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead)
	rbdFilesToRead = append(rbdFilesToRead, newFile)
	cephfsFilesToRead = append(cephfsFilesToRead, newFile)
	logger.Infof("Verified upgrade from v1.2 to master")
}

func (s *UpgradeSuite) gatherLogs(systemNamespace, testSuffix string) {
	// Gather logs before Ceph upgrade to help with debugging
	if installer.Env.Logs == "all" {
		s.k8sh.PrintPodDescribe(s.namespace)
	}
	n := strings.Replace(s.T().Name(), "/", "_", -1) + testSuffix
	s.op.installer.GatherAllRookLogs(n, systemNamespace, s.namespace)
}

func (s *UpgradeSuite) upgradeCephVersion(newCephImage string, numOSDs int) {
	osdDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd")<commit_msg>ceph: integration test for upgrade from nautilus to octopus<commit_after>/*
Copyright 2016 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package integration

import (
	"fmt"
	"strings"
	"testing"
	"time"

	"github.com/rook/rook/pkg/operator/k8sutil"
	"github.com/rook/rook/tests/framework/clients"
	"github.com/rook/rook/tests/framework/installer"
	"github.com/rook/rook/tests/framework/utils"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
)

const (
	rbdPodName        = "test-pod-upgrade"
	operatorContainer = "rook-ceph-operator"
)

// ************************************************
// *** Major scenarios tested by the UpgradeSuite ***
// Setup
// - Initially create a cluster from the previous minor release
// - Upgrade to the current build of Rook to verify functionality after upgrade
// - Test basic usage of block, object, and file after upgrade
// Monitors
// - One mon in the cluster
// ************************************************
func TestCephUpgradeSuite(t *testing.T) {
	if installer.SkipTestSuite(installer.CephTestSuite) {
		t.Skip()
	}

	// Skip the suite if CSI is not supported
	kh, err := utils.CreateK8sHelper(func() *testing.T { return t })
	require.NoError(t, err)
	checkSkipCSITest(t, kh)

	s := new(UpgradeSuite)
	defer func(s *UpgradeSuite) {
		HandlePanics(recover(), s.op, s.T)
	}(s)
	suite.Run(t, s)
}

type UpgradeSuite struct {
	suite.Suite
	helper    *clients.TestClient
	op        *TestCluster
	k8sh      *utils.K8sHelper
	namespace string
}

func (s *UpgradeSuite) SetupSuite() {
	s.namespace = "upgrade-ns"
	mons := 1
	rbdMirrorWorkers := 0
	s.op, s.k8sh = StartTestCluster(s.T,
		upgradeMinimalTestVersion,
		s.namespace,
		"",
		false,
		false,
		"",
		mons,
		rbdMirrorWorkers,
		installer.Version1_1,
		installer.MimicVersion,
		false,
	)
	s.helper = clients.CreateTestClient(s.k8sh, s.op.installer.Manifests)
}

func (s *UpgradeSuite) TearDownSuite() {
	s.op.Teardown()
}

func (s *UpgradeSuite) TestUpgradeToMaster() {
	systemNamespace := installer.SystemNamespace(s.namespace)

	//
	// Create block, object, and file storage before the upgrade
	//
	poolName := "upgradepool"
	storageClassName := "block-upgrade"
	blockName := "block-claim-upgrade"
	logger.Infof("Initializing block before the upgrade")
	setupBlockLite(s.helper, s.k8sh, s.Suite, s.namespace, systemNamespace, poolName, storageClassName, blockName, rbdPodName, s.op.installer.CephVersion)

	createPodWithBlock(s.helper, s.k8sh, s.Suite, s.namespace, storageClassName, rbdPodName, blockName)

	// FIX: We should require block images to be removed. See tracking issue:
	// <INSERT ISSUE>
	requireBlockImagesRemoved := false
	defer blockTestDataCleanUp(s.helper, s.k8sh, s.Suite, s.namespace, poolName, storageClassName, blockName, rbdPodName, requireBlockImagesRemoved)

	// Create the filesystem, but wait to create the file test client until we upgrade to nautilus
	logger.Infof("Initializing file before the upgrade")
	filesystemName := "upgrade-test-fs"
	activeCount := 1
	createFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, activeCount)

	logger.Infof("Initializing object before the upgrade")
	objectStoreName := "upgraded-object"
	runObjectE2ETestLite(s.helper, s.k8sh, s.Suite, s.namespace, objectStoreName, 1, false)

	// verify that we're actually running the right pre-upgrade image
	s.verifyOperatorImage(installer.Version1_1)

	message := "my simple message"
	preFilename := "pre-upgrade-file"
	assert.NoError(s.T(), s.k8sh.WriteToPod("", rbdPodName, preFilename, message))
	assert.NoError(s.T(), s.k8sh.ReadFromPod("", rbdPodName, preFilename, message))

	// we will keep appending to this to continue verifying old files through the upgrades
	rbdFilesToRead := []string{preFilename}
	cephfsFilesToRead := []string{}

	// Get some info about the currently deployed OSDs to determine later if they are all updated
	osdDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd")
	require.NoError(s.T(), err)
	osdDeps := osdDepList.Items
	numOSDs := len(osdDeps) // there should be this many upgraded OSDs
	require.NotEqual(s.T(), 0, numOSDs)

	//
	// Upgrade Rook from v1.1 to v1.2
	//
	logger.Infof("*** UPGRADING ROOK FROM v1.1 to v1.2 ***")
	s.gatherLogs(systemNamespace, "_before_1.2_upgrade")
	s.upgradeToV1_2()

	s.verifyOperatorImage(installer.Version1_2)
	s.verifyRookUpgrade(numOSDs)
	logger.Infof("Done with automatic upgrade from v1.1 to v1.2")
	newFile := "post-upgrade-1_1-to-1_2-file"
	s.verifyFilesAfterUpgrade("", newFile, message, rbdFilesToRead, cephfsFilesToRead)
	rbdFilesToRead = append(rbdFilesToRead, newFile)
	logger.Infof("Verified upgrade from v1.1 to v1.2")

	//
	// Upgrade from mimic to nautilus
	//
	logger.Infof("*** UPGRADING CEPH FROM Mimic TO Nautilus ***")
	s.gatherLogs(systemNamespace, "_before_nautilus_upgrade")
	s.upgradeCephVersion(installer.NautilusVersion.Image, numOSDs)

	// Start the file test client now that the CSI driver is supported on nautilus
	fsStorageClass := "file-upgrade"
	assert.NoError(s.T(), s.helper.FSClient.CreateStorageClass(filesystemName, s.namespace, fsStorageClass))
	useCSI := true
	createFilesystemConsumerPod(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName, fsStorageClass, useCSI)
	defer func() {
		cleanupFilesystemConsumer(s.helper, s.k8sh, s.Suite, s.namespace, filePodName)
		cleanupFilesystem(s.helper, s.k8sh, s.Suite, s.namespace, filesystemName)
	}()

	// Verify reading and writing to the test clients
	newFile = "post-nautilus-upgrade-file"
	s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead)
	rbdFilesToRead = append(rbdFilesToRead, newFile)
	cephfsFilesToRead = append(cephfsFilesToRead, newFile)
	logger.Infof("Verified upgrade from mimic to nautilus")

	//
	// Upgrade Rook from v1.2 to master
	//
	logger.Infof("*** UPGRADING ROOK FROM v1.2 to master ***")
	s.gatherLogs(systemNamespace, "_before_master_upgrade")
	s.upgradeToMaster()

	s.verifyOperatorImage(installer.VersionMaster)
	s.verifyRookUpgrade(numOSDs)
	logger.Infof("Done with automatic upgrade from v1.2 to master")
	newFile = "post-upgrade-1_2-to-master-file"
	s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead)
	rbdFilesToRead = append(rbdFilesToRead, newFile)
	cephfsFilesToRead = append(cephfsFilesToRead, newFile)
	logger.Infof("Verified upgrade from v1.2 to master")

	//
	// Upgrade from nautilus to octopus
	//
	logger.Infof("*** UPGRADING CEPH FROM Nautilus TO Octopus ***")
	s.gatherLogs(systemNamespace, "_before_octopus_upgrade")
	s.upgradeCephVersion(installer.OctopusVersion.Image, numOSDs)
	// Verify reading and writing to the test clients
	newFile = "post-octopus-upgrade-file"
	s.verifyFilesAfterUpgrade(filesystemName, newFile, message, rbdFilesToRead, cephfsFilesToRead)
	rbdFilesToRead = append(rbdFilesToRead, newFile)
	cephfsFilesToRead = append(cephfsFilesToRead, newFile)
	logger.Infof("Verified upgrade from nautilus to octopus")
}

func (s *UpgradeSuite) gatherLogs(systemNamespace, testSuffix string) {
	// Gather logs before Ceph upgrade to help with debugging
	if installer.Env.Logs == "all" {
		s.k8sh.PrintPodDescribe(s.namespace)
	}
	n := strings.Replace(s.T().Name(), "/", "_", -1) + testSuffix
	s.op.installer.GatherAllRookLogs(n, systemNamespace, s.namespace)
}

func (s *UpgradeSuite) upgradeCephVersion(newCephImage string, numOSDs int) {
	osdDepList, err := k8sutil.GetDeployments(s.k8sh.Clientset, s.namespace, "app=rook-ceph-osd")
{'fraction_non_alphanumeric': 0.12605420717182772, 'fraction_numerical': 0.009785617717118392, 'mean_word_length': 4.9540053660406285, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3418156', 'n_tokens_mistral': 5504, 'n_tokens_neox': 5307, 'n_words': 1312}