<commit_before>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.utils;

import com.yahoo.config.FileReference;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.producer.UserConfigRepo;
import java.util.logging.Level;
import com.yahoo.vespa.config.*;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.vespa.config.ConfigDefinition.DefaultValued;
import com.yahoo.vespa.model.AbstractService;

import java.io.Serializable;
import java.util.*;

/**
 * Utility methods for sending files to a collection of nodes.
 *
 * @author gjoranv
 */
public class FileSender implements Serializable {

    /**
     * Send the given file to all given services.
     *
     * @param relativePath  The path to the file, relative to the app pkg.
     * @param services  The services to send the file to.
     * @return The file reference that the file was given, never null.
     * @throws IllegalStateException if services is empty.
     */
    public static FileReference sendFileToServices(String relativePath,
                                                   Collection<? extends AbstractService> services) {
        if (services.isEmpty()) {
            throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " +
                                            "using 'count' instead of <node> tags.");
        }

        FileReference fileref = null;
        for (AbstractService service : services) {
            // The same reference will be returned from each call.
            fileref = service.sendFile(relativePath);
        }
        return fileref;
    }

    public static FileReference sendUriToServices(String uri, Collection<? extends AbstractService> services) {
        if (services.isEmpty()) {
            throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " +
                    "using 'count' instead of <node> tags.");
        }

        FileReference fileref = null;
        for (AbstractService service : services) {
            // The same reference will be returned from each call.
            fileref = service.sendUri(uri);
        }
        return fileref;
    }

    /**
     * Sends all user configured files for a producer to all given services.
     */
    public static <PRODUCER extends AbstractConfigProducer<?>>
    void sendUserConfiguredFiles(PRODUCER producer, Collection<? extends AbstractService> services, DeployLogger logger) {
        if (services.isEmpty())
            return;

        UserConfigRepo userConfigs = producer.getUserConfigs();
        Map<String, FileReference> sentFiles = new HashMap<>();
        for (ConfigDefinitionKey key : userConfigs.configsProduced()) {
            ConfigPayloadBuilder builder = userConfigs.get(key);
            try {
                sendUserConfiguredFiles(builder, sentFiles, services, key, logger);
            } catch (IllegalArgumentException e) {
                throw new IllegalArgumentException("Unable to send files for " + key, e);
            }
        }
    }

    private static void sendUserConfiguredFiles(ConfigPayloadBuilder builder,
                                                Map<String, FileReference> sentFiles,
                                                Collection<? extends AbstractService> services,
                                                ConfigDefinitionKey key, DeployLogger logger) {
        ConfigDefinition configDefinition = builder.getConfigDefinition();
        if (configDefinition == null) {
            // TODO: throw new IllegalArgumentException("Not able to find config definition for " + builder);
            logger.log(Level.WARNING, "Not able to find config definition for " + key + ". Will not send files for this config");
            return;
        }
        // Inspect fields at this level
        sendEntries(builder, sentFiles, services, configDefinition.getFileDefs());
        sendEntries(builder, sentFiles, services, configDefinition.getPathDefs());

        // Inspect arrays
        for (Map.Entry<String, ConfigDefinition.ArrayDef> entry : configDefinition.getArrayDefs().entrySet()) {
            if (isFileOrPathArray(entry)) {
                ConfigPayloadBuilder.Array array = builder.getArray(entry.getKey());
                sendFileEntries(array.getElements(), sentFiles, services);
            }
        }
        // Maps
        for (Map.Entry<String, ConfigDefinition.LeafMapDef> entry : configDefinition.getLeafMapDefs().entrySet()) {
            if (isFileOrPathMap(entry)) {
                ConfigPayloadBuilder.MapBuilder map = builder.getMap(entry.getKey());
                sendFileEntries(map.getElements(), sentFiles, services);
            }
        }

        // Inspect inner fields
        for (String name : configDefinition.getStructDefs().keySet()) {
            sendUserConfiguredFiles(builder.getObject(name), sentFiles, services, key, logger);<commit_msg>Reduce log level<commit_after>// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.model.utils;

import com.yahoo.config.FileReference;
import com.yahoo.config.application.api.DeployLogger;
import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.config.model.producer.UserConfigRepo;
import com.yahoo.vespa.config.ConfigDefinition;
import com.yahoo.vespa.config.ConfigDefinition.DefaultValued;
import com.yahoo.vespa.config.ConfigDefinitionKey;
import com.yahoo.vespa.config.ConfigPayloadBuilder;
import com.yahoo.vespa.model.AbstractService;

import java.io.Serializable;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.logging.Level;

/**
 * Utility methods for sending files to a collection of nodes.
 *
 * @author gjoranv
 */
public class FileSender implements Serializable {

    /**
     * Send the given file to all given services.
     *
     * @param relativePath  The path to the file, relative to the app pkg.
     * @param services  The services to send the file to.
     * @return The file reference that the file was given, never null.
     * @throws IllegalStateException if services is empty.
     */
    public static FileReference sendFileToServices(String relativePath,
                                                   Collection<? extends AbstractService> services) {
        if (services.isEmpty()) {
            throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " +
                                            "using 'count' instead of <node> tags.");
        }

        FileReference fileref = null;
        for (AbstractService service : services) {
            // The same reference will be returned from each call.
            fileref = service.sendFile(relativePath);
        }
        return fileref;
    }

    public static FileReference sendUriToServices(String uri, Collection<? extends AbstractService> services) {
        if (services.isEmpty()) {
            throw new IllegalStateException("No service instances. Probably a standalone cluster setting up <nodes> " +
                    "using 'count' instead of <node> tags.");
        }

        FileReference fileref = null;
        for (AbstractService service : services) {
            // The same reference will be returned from each call.
            fileref = service.sendUri(uri);
        }
        return fileref;
    }

    /**
     * Sends all user configured files for a producer to all given services.
     */
    public static <PRODUCER extends AbstractConfigProducer<?>>
    void sendUserConfiguredFiles(PRODUCER producer, Collection<? extends AbstractService> services, DeployLogger logger) {
        if (services.isEmpty())
            return;

        UserConfigRepo userConfigs = producer.getUserConfigs();
        Map<String, FileReference> sentFiles = new HashMap<>();
        for (ConfigDefinitionKey key : userConfigs.configsProduced()) {
            ConfigPayloadBuilder builder = userConfigs.get(key);
            try {
                sendUserConfiguredFiles(builder, sentFiles, services, key, logger);
            } catch (IllegalArgumentException e) {
                throw new IllegalArgumentException("Unable to send files for " + key, e);
            }
        }
    }

    private static void sendUserConfiguredFiles(ConfigPayloadBuilder builder,
                                                Map<String, FileReference> sentFiles,
                                                Collection<? extends AbstractService> services,
                                                ConfigDefinitionKey key, DeployLogger logger) {
        ConfigDefinition configDefinition = builder.getConfigDefinition();
        if (configDefinition == null) {
            // TODO: throw new IllegalArgumentException("Not able to find config definition for " + builder);
            logger.log(Level.FINE, "Not able to find config definition for " + key + ". Will not send files for this config");
            return;
        }
        // Inspect fields at this level
        sendEntries(builder, sentFiles, services, configDefinition.getFileDefs());
        sendEntries(builder, sentFiles, services, configDefinition.getPathDefs());

        // Inspect arrays
        for (Map.Entry<String, ConfigDefinition.ArrayDef> entry : configDefinition.getArrayDefs().entrySet()) {
            if (isFileOrPathArray(entry)) {
                ConfigPayloadBuilder.Array array = builder.getArray(entry.getKey());
                sendFileEntries(array.getElements(), sentFiles, services);
            }
        }
        // Maps
        for (Map.Entry<String, ConfigDefinition.LeafMapDef> entry : configDefinition.getLeafMapDefs().entrySet()) {
            if (isFileOrPathMap(entry)) {
                ConfigPayloadBuilder.MapBuilder map = builder.getMap(entry.getKey());
                sendFileEntries(map.getElements(), sentFiles, services);
            }
        }

        // Inspect inner fields
        for (String name : configDefinition.getStructDefs().keySet()) {
            sendUserConfiguredFiles(builder.getObject(name), sentFiles, services, key, logger);
{'fraction_non_alphanumeric': 0.08546348991993827, 'fraction_numerical': 0.0011575190508343782, 'mean_word_length': 2.4274380165289258, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 20, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7250935', 'n_tokens_mistral': 2498, 'n_tokens_neox': 2418, 'n_words': 747}