<commit_before>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.maintenance;

import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Deployer;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.jdisc.Metric;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeList;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.maintenance.MaintenanceDeployment.Move;
import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.provisioning.HostCapacity;
import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceComparator;

import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.logging.Level;
import java.util.stream.Collectors;

/**
 * A maintainer which attempts to ensure there is spare capacity available in chunks which can fit
 * all node resource configuration in use, such that the system is able to quickly replace a failed node
 * if necessary.
 *
 * This also emits the following metrics:
 * - Overcommitted hosts: Hosts whose capacity is less than the sum of its children's
 * - Spare host capacity, or how many hosts the repository can stand to lose without ending up in a situation where it's
 *   unable to find a new home for orphaned tenants.
 *
 * @author mgimle
 * @author bratseth
 */
public class SpareCapacityMaintainer extends NodeRepositoryMaintainer {

    private final int maxIterations;
    private final Deployer deployer;
    private final Metric metric;

    public SpareCapacityMaintainer(Deployer deployer,
                                   NodeRepository nodeRepository,
                                   Metric metric,
                                   Duration interval) {
        this(deployer, nodeRepository, metric, interval,
             10_000 // Should take less than a few minutes
            );
    }

    public SpareCapacityMaintainer(Deployer deployer,
                                   NodeRepository nodeRepository,
                                   Metric metric,
                                   Duration interval,
                                   int maxIterations) {
        super(nodeRepository, interval, metric);
        this.deployer = deployer;
        this.metric = metric;
        this.maxIterations = maxIterations;
    }

    @Override
    protected boolean maintain() {
        if ( ! nodeRepository().isWorking()) return false;

        boolean success = true;
        // Don't need to maintain spare capacity in dynamically provisioned zones; can provision more on demand.
        if (nodeRepository().zone().getCloud().dynamicProvisioning()) return success;

        NodeList allNodes = nodeRepository().list();
        CapacityChecker capacityChecker = new CapacityChecker(allNodes);

        List<Node> overcommittedHosts = capacityChecker.findOvercommittedHosts();
        metric.set("overcommittedHosts", overcommittedHosts.size(), null);
        retireOvercommitedHosts(allNodes, overcommittedHosts);

        Optional<CapacityChecker.HostFailurePath> failurePath = capacityChecker.worstCaseHostLossLeadingToFailure();
        if (failurePath.isPresent()) {
            int spareHostCapacity = failurePath.get().hostsCausingFailure.size() - 1;
            if (spareHostCapacity == 0) {
                List<Move> mitigation = findMitigation(failurePath.get());
                if (execute(mitigation)) {
                    // We succeeded or are in the process of taking a step to mitigate.
                    // Report with the assumption this will eventually succeed to avoid alerting before we're stuck
                    spareHostCapacity++;
                }
                else {
                    success = false;
                }
            }
            metric.set("spareHostCapacity", spareHostCapacity, null);
        }
        return success;
    }

    private boolean execute(List<Move> mitigation) {
        if (mitigation.isEmpty()) {
            log.warning("Out of spare capacity. No mitigation could be found");
            return false;
        }
        Move firstMove = mitigation.get(0);
        if (firstMove.node().allocation().get().membership().retired()) return true; // Already in progress
        boolean success = firstMove.execute(false, Agent.SpareCapacityMaintainer, deployer, metric, nodeRepository());
        log.info("Out of spare capacity. Mitigation plan: " + mitigation + ". First move successful: " + success);
        return success;
    }

    private List<Move> findMitigation(CapacityChecker.HostFailurePath failurePath) {
        Optional<Node> nodeWhichCantMove = failurePath.failureReason.tenant;
        if (nodeWhichCantMove.isEmpty()) return List.of();<commit_msg>Log out of spare details<commit_after>// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.provision.maintenance;

import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Deployer;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.jdisc.Metric;
import com.yahoo.vespa.hosted.provision.Node;
import com.yahoo.vespa.hosted.provision.NodeList;
import com.yahoo.vespa.hosted.provision.NodeRepository;
import com.yahoo.vespa.hosted.provision.maintenance.MaintenanceDeployment.Move;
import com.yahoo.vespa.hosted.provision.node.Agent;
import com.yahoo.vespa.hosted.provision.provisioning.HostCapacity;
import com.yahoo.vespa.hosted.provision.provisioning.NodeResourceComparator;

import java.time.Duration;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.logging.Level;
import java.util.stream.Collectors;

/**
 * A maintainer which attempts to ensure there is spare capacity available in chunks which can fit
 * all node resource configuration in use, such that the system is able to quickly replace a failed node
 * if necessary.
 *
 * This also emits the following metrics:
 * - Overcommitted hosts: Hosts whose capacity is less than the sum of its children's
 * - Spare host capacity, or how many hosts the repository can stand to lose without ending up in a situation where it's
 *   unable to find a new home for orphaned tenants.
 *
 * @author mgimle
 * @author bratseth
 */
public class SpareCapacityMaintainer extends NodeRepositoryMaintainer {

    private final int maxIterations;
    private final Deployer deployer;
    private final Metric metric;

    public SpareCapacityMaintainer(Deployer deployer,
                                   NodeRepository nodeRepository,
                                   Metric metric,
                                   Duration interval) {
        this(deployer, nodeRepository, metric, interval,
             10_000 // Should take less than a few minutes
            );
    }

    public SpareCapacityMaintainer(Deployer deployer,
                                   NodeRepository nodeRepository,
                                   Metric metric,
                                   Duration interval,
                                   int maxIterations) {
        super(nodeRepository, interval, metric);
        this.deployer = deployer;
        this.metric = metric;
        this.maxIterations = maxIterations;
    }

    @Override
    protected boolean maintain() {
        if ( ! nodeRepository().isWorking()) return false;

        boolean success = true;
        // Don't need to maintain spare capacity in dynamically provisioned zones; can provision more on demand.
        if (nodeRepository().zone().getCloud().dynamicProvisioning()) return success;

        NodeList allNodes = nodeRepository().list();
        CapacityChecker capacityChecker = new CapacityChecker(allNodes);

        List<Node> overcommittedHosts = capacityChecker.findOvercommittedHosts();
        metric.set("overcommittedHosts", overcommittedHosts.size(), null);
        retireOvercommitedHosts(allNodes, overcommittedHosts);

        Optional<CapacityChecker.HostFailurePath> failurePath = capacityChecker.worstCaseHostLossLeadingToFailure();
        if (failurePath.isPresent()) {
            int spareHostCapacity = failurePath.get().hostsCausingFailure.size() - 1;
            if (spareHostCapacity == 0) {
                List<Move> mitigation = findMitigation(failurePath.get());
                if (execute(mitigation, failurePath.get())) {
                    // We succeeded or are in the process of taking a step to mitigate.
                    // Report with the assumption this will eventually succeed to avoid alerting before we're stuck
                    spareHostCapacity++;
                }
                else {
                    success = false;
                }
            }
            metric.set("spareHostCapacity", spareHostCapacity, null);
        }
        return success;
    }

    private boolean execute(List<Move> mitigation, CapacityChecker.HostFailurePath failurePath) {
        if (mitigation.isEmpty()) {
            log.warning("Out of spare capacity and no mitigation possible: " + failurePath);
            return false;
        }
        Move firstMove = mitigation.get(0);
        if (firstMove.node().allocation().get().membership().retired()) return true; // Already in progress
        boolean success = firstMove.execute(false, Agent.SpareCapacityMaintainer, deployer, metric, nodeRepository());
        log.info("Out of spare capacity. Mitigation plan: " + mitigation + ". First move successful: " + success);
        return success;
    }

    private List<Move> findMitigation(CapacityChecker.HostFailurePath failurePath) {
        Optional<Node> nodeWhichCantMove = failurePath.failureReason.tenant;
        if (nodeWhichCantMove.isEmpty()) return List.of();
{'fraction_non_alphanumeric': 0.08265345765345765, 'fraction_numerical': 0.0019425019425019425, 'mean_word_length': 2.7106306306306305, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '265219', 'n_tokens_mistral': 2689, 'n_tokens_neox': 2565, 'n_words': 746}