<commit_before>/*
Copyright 2019 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package clusterdisruption

import (
	"context"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/rook/rook/pkg/clusterd"
	cephClient "github.com/rook/rook/pkg/daemon/ceph/client"
	"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
	"github.com/rook/rook/pkg/operator/k8sutil"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

func (r *ReconcileClusterDisruption) getOsdDataList(request reconcile.Request, poolFailureDomain string) ([]OsdData, error) {
	osdDeploymentList := &appsv1.DeploymentList{}
	namespaceListOpts := client.InNamespace(request.Namespace)
	err := r.client.List(context.TODO(), osdDeploymentList, client.MatchingLabels{k8sutil.AppAttr: osd.AppName}, namespaceListOpts)
	if err != nil {
		return nil, fmt.Errorf("could not list osd deployments: %+v", err)
	}

	osds := make([]OsdData, 0)

	for _, deployment := range osdDeploymentList.Items {
		osdData := OsdData{Deployment: deployment}
		labels := deployment.Spec.Template.ObjectMeta.GetLabels()
		osdID, ok := labels[osd.OsdIdLabelKey]
		if !ok {
			return nil, fmt.Errorf("osd %q was not labeled", deployment.ObjectMeta.Name)
		}
		osdIDInt, err := strconv.Atoi(osdID)
		if err != nil {
			return nil, fmt.Errorf("failed to convert osd id %q in an int. %+v", osdID, err)
		}
		crushMeta, err := r.osdCrushLocationMap.Get(request.Namespace, osdIDInt)
		if err != nil {
			// If the error contains that message, this means the cluster is not up and running
			// No monitors are present and thus no ceph configuration has been created
			//
			// Or this means the ceph config hasn't been written on the operator yet
			// The controller starts before we run WriteConnectionConfig()
			if strings.Contains(err.Error(), "error calling conf_read_file") {
				logger.Debugf("Ceph %q cluster is not ready, cannot check osd location yet.", request.Namespace)
				return nil, nil
			}
			return nil, fmt.Errorf("could not fetch info from ceph for osd %q. %+v", osdID, err)
		}
		// bypass the cache if the topology location is not populated in the cache
		_, failureDomainKnown := crushMeta.Location[poolFailureDomain]
		if !failureDomainKnown {
			crushMeta, err = r.osdCrushLocationMap.get(request.Namespace, osdIDInt)
			if err != nil {
				return nil, fmt.Errorf("could not fetch info from ceph for osd %q. %+v", osdID, err)
			}
		}

		osdData.CrushMeta = crushMeta
		osds = append(osds, osdData)

	}
	return osds, nil
}

// OsdData stores the deployment and the Crush Data of the osd together
type OsdData struct {
	Deployment appsv1.Deployment
	CrushMeta  *cephClient.CrushFindResult
}

// OSDCrushLocationMap is used to maintain a cache of map of osd id to cephClientCrushhFindResults
// the crush position of osds wrt to the failureDomain is not expected to change often, so a default Resync period
// of half an hour is used, but if a use case arises where this is needed, ResyncPeriod should be made smaller.
type OSDCrushLocationMap struct {
	ResyncPeriod       time.Duration
	Context            *clusterd.Context
	clusterLocationMap map[string]map[int]cachedOSDLocation
	mux                sync.Mutex
}

type cachedOSDLocation struct {
	result     *cephClient.CrushFindResult
	lastSynced time.Time
}

// Get takes an osd id and returns a CrushFindResult from cache
func (o *OSDCrushLocationMap) Get(clusterNamespace string, id int) (*cephClient.CrushFindResult, error) {
	o.mux.Lock()
	defer o.mux.Unlock()
	if o.ResyncPeriod == 0 {
		o.ResyncPeriod = 30 * time.Minute
	}

	// initialize clusterLocationMap
	if len(o.clusterLocationMap) == 0 {
		o.clusterLocationMap = make(map[string]map[int]cachedOSDLocation)
	}
	locationMap, ok := o.clusterLocationMap[clusterNamespace]
	// initialize namespace map
	if !ok {
		o.clusterLocationMap[clusterNamespace] = make(map[int]cachedOSDLocation)
	}

	// sync of osd id not found in clusterNamespace
	osdLocation, ok := locationMap[id]
	if !ok {
		osdResult, err := o.get(clusterNamespace, id)
		if err != nil {
			return nil, fmt.Errorf("failed to run `find` on osd %d in cluster %s: %v", id, clusterNamespace, err)
		}
		o.clusterLocationMap[clusterNamespace][id] = cachedOSDLocation{result: osdResult, lastSynced: time.Now()}
		return osdResult, nil
	}

	// sync if not synced for longer than ResyncPeriod
	if time.Since(osdLocation.lastSynced) > o.ResyncPeriod {
		osdResult, err := o.get(clusterNamespace, id)
		if err != nil {
			return nil, fmt.Errorf("failed to run `find` on osd %d in cluster %s: %v", id, clusterNamespace, err)
		}
		o.clusterLocationMap[clusterNamespace][id] = cachedOSDLocation{result: osdResult, lastSynced: time.Now()}
		return osdResult, nil
	}

	return osdLocation.result, nil

}

// uncached version
func (o *OSDCrushLocationMap) get(clusterNamespace string, id int) (*cephClient.CrushFindResult, error) {
	osdResult, err := cephClient.FindOSDInCrushMap(o.Context, clusterNamespace, id)
	if err != nil {
		return nil, fmt.Errorf("failed running find on osd %d: %+v", id, err)
	}
	o.clusterLocationMap[clusterNamespace][id] = cachedOSDLocation{
		result:     osdResult,
		lastSynced: time.Now(),
	}
	return osdResult, nil
}

func getOSDsForNodes(osdDataList []OsdData, nodeList []*corev1.Node, failureDomainType string) ([]OsdData, error) {
	nodeOsdDataList := make([]OsdData, 0)
	for _, node := range nodeList {
		if node == nil {
			logger.Warningf("node in nodelist was nil")
			continue
		}<commit_msg>ceph: print the full err fields<commit_after>/*
Copyright 2019 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package clusterdisruption

import (
	"context"
	"fmt"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/rook/rook/pkg/clusterd"
	cephClient "github.com/rook/rook/pkg/daemon/ceph/client"
	"github.com/rook/rook/pkg/operator/ceph/cluster/osd"
	"github.com/rook/rook/pkg/operator/k8sutil"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

func (r *ReconcileClusterDisruption) getOsdDataList(request reconcile.Request, poolFailureDomain string) ([]OsdData, error) {
	osdDeploymentList := &appsv1.DeploymentList{}
	namespaceListOpts := client.InNamespace(request.Namespace)
	err := r.client.List(context.TODO(), osdDeploymentList, client.MatchingLabels{k8sutil.AppAttr: osd.AppName}, namespaceListOpts)
	if err != nil {
		return nil, fmt.Errorf("could not list osd deployments: %+v", err)
	}

	osds := make([]OsdData, 0)

	for _, deployment := range osdDeploymentList.Items {
		osdData := OsdData{Deployment: deployment}
		labels := deployment.Spec.Template.ObjectMeta.GetLabels()
		osdID, ok := labels[osd.OsdIdLabelKey]
		if !ok {
			return nil, fmt.Errorf("osd %q was not labeled", deployment.ObjectMeta.Name)
		}
		osdIDInt, err := strconv.Atoi(osdID)
		if err != nil {
			return nil, fmt.Errorf("failed to convert osd id %q in an int. %+v", osdID, err)
		}
		crushMeta, err := r.osdCrushLocationMap.Get(request.Namespace, osdIDInt)
		if err != nil {
			// If the error contains that message, this means the cluster is not up and running
			// No monitors are present and thus no ceph configuration has been created
			//
			// Or this means the ceph config hasn't been written on the operator yet
			// The controller starts before we run WriteConnectionConfig()
			if strings.Contains(err.Error(), "error calling conf_read_file") {
				logger.Debugf("Ceph %q cluster is not ready, cannot check osd location yet.", request.Namespace)
				return nil, nil
			}
			return nil, fmt.Errorf("could not fetch info from ceph for osd %q. %+v", osdID, err)
		}
		// bypass the cache if the topology location is not populated in the cache
		_, failureDomainKnown := crushMeta.Location[poolFailureDomain]
		if !failureDomainKnown {
			crushMeta, err = r.osdCrushLocationMap.get(request.Namespace, osdIDInt)
			if err != nil {
				return nil, fmt.Errorf("could not fetch info from ceph for osd %q. %+v", osdID, err)
			}
		}

		osdData.CrushMeta = crushMeta
		osds = append(osds, osdData)

	}
	return osds, nil
}

// OsdData stores the deployment and the Crush Data of the osd together
type OsdData struct {
	Deployment appsv1.Deployment
	CrushMeta  *cephClient.CrushFindResult
}

// OSDCrushLocationMap is used to maintain a cache of map of osd id to cephClientCrushhFindResults
// the crush position of osds wrt to the failureDomain is not expected to change often, so a default Resync period
// of half an hour is used, but if a use case arises where this is needed, ResyncPeriod should be made smaller.
type OSDCrushLocationMap struct {
	ResyncPeriod       time.Duration
	Context            *clusterd.Context
	clusterLocationMap map[string]map[int]cachedOSDLocation
	mux                sync.Mutex
}

type cachedOSDLocation struct {
	result     *cephClient.CrushFindResult
	lastSynced time.Time
}

// Get takes an osd id and returns a CrushFindResult from cache
func (o *OSDCrushLocationMap) Get(clusterNamespace string, id int) (*cephClient.CrushFindResult, error) {
	o.mux.Lock()
	defer o.mux.Unlock()
	if o.ResyncPeriod == 0 {
		o.ResyncPeriod = 30 * time.Minute
	}

	// initialize clusterLocationMap
	if len(o.clusterLocationMap) == 0 {
		o.clusterLocationMap = make(map[string]map[int]cachedOSDLocation)
	}
	locationMap, ok := o.clusterLocationMap[clusterNamespace]
	// initialize namespace map
	if !ok {
		o.clusterLocationMap[clusterNamespace] = make(map[int]cachedOSDLocation)
	}

	// sync of osd id not found in clusterNamespace
	osdLocation, ok := locationMap[id]
	if !ok {
		osdResult, err := o.get(clusterNamespace, id)
		if err != nil {
			return nil, fmt.Errorf("failed to run `find` on osd %d in cluster %s. %+v", id, clusterNamespace, err)
		}
		o.clusterLocationMap[clusterNamespace][id] = cachedOSDLocation{result: osdResult, lastSynced: time.Now()}
		return osdResult, nil
	}

	// sync if not synced for longer than ResyncPeriod
	if time.Since(osdLocation.lastSynced) > o.ResyncPeriod {
		osdResult, err := o.get(clusterNamespace, id)
		if err != nil {
			return nil, fmt.Errorf("failed to run `find` on osd %d in cluster %s. %+v", id, clusterNamespace, err)
		}
		o.clusterLocationMap[clusterNamespace][id] = cachedOSDLocation{result: osdResult, lastSynced: time.Now()}
		return osdResult, nil
	}

	return osdLocation.result, nil

}

// uncached version
func (o *OSDCrushLocationMap) get(clusterNamespace string, id int) (*cephClient.CrushFindResult, error) {
	osdResult, err := cephClient.FindOSDInCrushMap(o.Context, clusterNamespace, id)
	if err != nil {
		return nil, fmt.Errorf("failed running find on osd %d: %+v", id, err)
	}
	o.clusterLocationMap[clusterNamespace][id] = cachedOSDLocation{
		result:     osdResult,
		lastSynced: time.Now(),
	}
	return osdResult, nil
}

func getOSDsForNodes(osdDataList []OsdData, nodeList []*corev1.Node, failureDomainType string) ([]OsdData, error) {
	nodeOsdDataList := make([]OsdData, 0)
	for _, node := range nodeList {
		if node == nil {
			logger.Warningf("node in nodelist was nil")
			continue
		}
{'fraction_non_alphanumeric': 0.10454993429697766, 'fraction_numerical': 0.004434954007884363, 'mean_word_length': 4.275996533795494, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '529145', 'n_tokens_mistral': 4244, 'n_tokens_neox': 4008, 'n_words': 1225}