<commit_before>// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build.buildfarm.server;

import static build.buildfarm.common.io.Utils.formatIOError;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.util.concurrent.MoreExecutors.shutdownAndAwaitTermination;
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
import static java.util.logging.Level.SEVERE;

import build.buildfarm.common.LoggingMain;
import build.buildfarm.common.grpc.TracingMetadataUtils.ServerHeadersInterceptor;
import build.buildfarm.metrics.MetricsPublisher;
import build.buildfarm.metrics.aws.AwsMetricsPublisher;
import build.buildfarm.metrics.gcp.GcpMetricsPublisher;
import build.buildfarm.metrics.log.LogMetricsPublisher;
import build.buildfarm.metrics.prometheus.PrometheusPublisher;
import build.buildfarm.v1test.BuildFarmServerConfig;
import build.buildfarm.v1test.MetricsConfig;
import com.google.devtools.common.options.OptionsParser;
import com.google.protobuf.TextFormat;
import io.grpc.Server;
import io.grpc.ServerBuilder;
import io.grpc.ServerInterceptor;
import io.grpc.health.v1.HealthCheckResponse.ServingStatus;
import io.grpc.services.HealthStatusManager;
import io.grpc.util.TransmitStatusRuntimeExceptionInterceptor;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.naming.ConfigurationException;

public class BuildFarmServer extends LoggingMain {
  // We need to keep references to the root and netty loggers to prevent them from being garbage
  // collected, which would cause us to loose their configuration.
  private static final java.util.logging.Logger nettyLogger =
      java.util.logging.Logger.getLogger("io.grpc.netty");
  private static final Logger logger = Logger.getLogger(BuildFarmServer.class.getName());

  private final ScheduledExecutorService keepaliveScheduler = newSingleThreadScheduledExecutor();
  private final ActionCacheRequestCounter actionCacheRequestCounter;
  private final Instances instances;
  private final HealthStatusManager healthStatusManager;
  private final Server server;
  private boolean stopping = false;

  public BuildFarmServer(String session, BuildFarmServerConfig config)
      throws InterruptedException, ConfigurationException {
    this(session, ServerBuilder.forPort(config.getPort()), config);
  }

  public BuildFarmServer(
      String session, ServerBuilder<?> serverBuilder, BuildFarmServerConfig config)
      throws InterruptedException, ConfigurationException {
    super("BuildFarmServer");
    String defaultInstanceName = config.getDefaultInstanceName();
    instances =
        new BuildFarmInstances(session, config.getInstancesList(), defaultInstanceName, this::stop);

    healthStatusManager = new HealthStatusManager();
    actionCacheRequestCounter =
        new ActionCacheRequestCounter(ActionCacheService.logger, Duration.ofSeconds(10));

    ServerInterceptor headersInterceptor = new ServerHeadersInterceptor();

    server =
        serverBuilder
            .addService(healthStatusManager.getHealthService())
            .addService(new ActionCacheService(instances, actionCacheRequestCounter::increment))
            .addService(new CapabilitiesService(instances))
            .addService(
                new ContentAddressableStorageService(
                    instances,
                    /* deadlineAfter=*/ 1,
                    TimeUnit.DAYS,
                    /* requestLogLevel=*/ Level.INFO))
            .addService(new ByteStreamService(instances, /* writeDeadlineAfter=*/ 1, TimeUnit.DAYS))
            .addService(
                new ExecutionService(
                    instances,
                    config.getExecuteKeepaliveAfterSeconds(),
                    TimeUnit.SECONDS,
                    keepaliveScheduler,
                    getMetricsPublisher(config.getMetricsConfig())))
            .addService(new OperationQueueService(instances))
            .addService(new OperationsService(instances))
            .addService(new AdminService(config.getAdminConfig(), instances))
            .addService(new FetchService(instances))
            .intercept(TransmitStatusRuntimeExceptionInterceptor.instance())
            .intercept(headersInterceptor)
            .build();

    logger.log(Level.INFO, String.format("%s initialized", session));
  }

  private static BuildFarmServerConfig toBuildFarmServerConfig(
      Readable input, BuildFarmServerOptions options) throws IOException {
    BuildFarmServerConfig.Builder builder = BuildFarmServerConfig.newBuilder();
    TextFormat.merge(input, builder);
    if (options.port > 0) {
      builder.setPort(options.port);
    }
    return builder.build();
  }

  private static MetricsPublisher getMetricsPublisher(MetricsConfig metricsConfig) {
    switch (metricsConfig.getMetricsDestination()) {
      default:
        return new LogMetricsPublisher(metricsConfig);
      case "aws":
        return new AwsMetricsPublisher(metricsConfig);
      case "gcp":
        return new GcpMetricsPublisher(metricsConfig);<commit_msg>Add grpc reflection service (#687)<commit_after>// Copyright 2017 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build.buildfarm.server;

import static build.buildfarm.common.io.Utils.formatIOError;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.util.concurrent.MoreExecutors.shutdownAndAwaitTermination;
import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
import static java.util.logging.Level.SEVERE;

import build.buildfarm.common.LoggingMain;
import build.buildfarm.common.grpc.TracingMetadataUtils.ServerHeadersInterceptor;
import build.buildfarm.metrics.MetricsPublisher;
import build.buildfarm.metrics.aws.AwsMetricsPublisher;
import build.buildfarm.metrics.gcp.GcpMetricsPublisher;
import build.buildfarm.metrics.log.LogMetricsPublisher;
import build.buildfarm.metrics.prometheus.PrometheusPublisher;
import build.buildfarm.v1test.BuildFarmServerConfig;
import build.buildfarm.v1test.MetricsConfig;
import com.google.devtools.common.options.OptionsParser;
import com.google.protobuf.TextFormat;
import io.grpc.Server;
import io.grpc.ServerBuilder;
import io.grpc.ServerInterceptor;
import io.grpc.health.v1.HealthCheckResponse.ServingStatus;
import io.grpc.protobuf.services.ProtoReflectionService;
import io.grpc.services.HealthStatusManager;
import io.grpc.util.TransmitStatusRuntimeExceptionInterceptor;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.naming.ConfigurationException;

public class BuildFarmServer extends LoggingMain {
  // We need to keep references to the root and netty loggers to prevent them from being garbage
  // collected, which would cause us to loose their configuration.
  private static final java.util.logging.Logger nettyLogger =
      java.util.logging.Logger.getLogger("io.grpc.netty");
  private static final Logger logger = Logger.getLogger(BuildFarmServer.class.getName());

  private final ScheduledExecutorService keepaliveScheduler = newSingleThreadScheduledExecutor();
  private final ActionCacheRequestCounter actionCacheRequestCounter;
  private final Instances instances;
  private final HealthStatusManager healthStatusManager;
  private final Server server;
  private boolean stopping = false;

  public BuildFarmServer(String session, BuildFarmServerConfig config)
      throws InterruptedException, ConfigurationException {
    this(session, ServerBuilder.forPort(config.getPort()), config);
  }

  public BuildFarmServer(
      String session, ServerBuilder<?> serverBuilder, BuildFarmServerConfig config)
      throws InterruptedException, ConfigurationException {
    super("BuildFarmServer");
    String defaultInstanceName = config.getDefaultInstanceName();
    instances =
        new BuildFarmInstances(session, config.getInstancesList(), defaultInstanceName, this::stop);

    healthStatusManager = new HealthStatusManager();
    actionCacheRequestCounter =
        new ActionCacheRequestCounter(ActionCacheService.logger, Duration.ofSeconds(10));

    ServerInterceptor headersInterceptor = new ServerHeadersInterceptor();

    server =
        serverBuilder
            .addService(healthStatusManager.getHealthService())
            .addService(new ActionCacheService(instances, actionCacheRequestCounter::increment))
            .addService(new CapabilitiesService(instances))
            .addService(
                new ContentAddressableStorageService(
                    instances,
                    /* deadlineAfter=*/ 1,
                    TimeUnit.DAYS,
                    /* requestLogLevel=*/ Level.INFO))
            .addService(new ByteStreamService(instances, /* writeDeadlineAfter=*/ 1, TimeUnit.DAYS))
            .addService(
                new ExecutionService(
                    instances,
                    config.getExecuteKeepaliveAfterSeconds(),
                    TimeUnit.SECONDS,
                    keepaliveScheduler,
                    getMetricsPublisher(config.getMetricsConfig())))
            .addService(new OperationQueueService(instances))
            .addService(new OperationsService(instances))
            .addService(new AdminService(config.getAdminConfig(), instances))
            .addService(new FetchService(instances))
            .addService(ProtoReflectionService.newInstance())
            .intercept(TransmitStatusRuntimeExceptionInterceptor.instance())
            .intercept(headersInterceptor)
            .build();

    logger.log(Level.INFO, String.format("%s initialized", session));
  }

  private static BuildFarmServerConfig toBuildFarmServerConfig(
      Readable input, BuildFarmServerOptions options) throws IOException {
    BuildFarmServerConfig.Builder builder = BuildFarmServerConfig.newBuilder();
    TextFormat.merge(input, builder);
    if (options.port > 0) {
      builder.setPort(options.port);
    }
    return builder.build();
  }

  private static MetricsPublisher getMetricsPublisher(MetricsConfig metricsConfig) {
    switch (metricsConfig.getMetricsDestination()) {
      default:
        return new LogMetricsPublisher(metricsConfig);
      case "aws":
        return new AwsMetricsPublisher(metricsConfig);
      case "gcp":
        return new GcpMetricsPublisher(metricsConfig);
{'fraction_non_alphanumeric': 0.08277331584986068, 'fraction_numerical': 0.0028683822324209146, 'mean_word_length': 3.9930441898527005, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2366477', 'n_tokens_mistral': 3148, 'n_tokens_neox': 3072, 'n_words': 740}