<commit_before>/*
 * Copyright 2019 The Bazel Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.idea.blaze.base.command.buildresult;

import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableSet.toImmutableSet;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.Queues;
import com.google.common.collect.SetMultimap;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildEventId.NamedSetOfFilesId;
import com.google.idea.blaze.base.model.primitives.Label;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Queue;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

/**
 * An intermediate data class representing blaze's build event protocol (BEP) output for a build.
 */
final class ParsedBepOutput {

  static ParsedBepOutput parseBepArtifacts(InputStream bepStream) throws IOException {
    BuildEventStreamProtos.BuildEvent event;
    Map<String, String> configIdToMnemonic = new HashMap<>();
    Map<String, BuildEventStreamProtos.NamedSetOfFiles> fileSets = new LinkedHashMap<>();
    Map<String, String> fileSetConfigs = new HashMap<>();
    ImmutableSetMultimap.Builder<String, String> outputGroupToFileSets =
        ImmutableSetMultimap.builder();
    ImmutableSetMultimap.Builder<String, String> targetToFileSets = ImmutableSetMultimap.builder();
    long startTimeMillis = 0L;

    while ((event = BuildEventStreamProtos.BuildEvent.parseDelimitedFrom(bepStream)) != null) {
      switch (event.getId().getIdCase()) {
        case CONFIGURATION:
          configIdToMnemonic.put(
              event.getId().getConfiguration().getId(), event.getConfiguration().getMnemonic());
          continue;
        case NAMED_SET:
          fileSets.put(event.getId().getNamedSet().getId(), event.getNamedSetOfFiles());
          continue;
        case TARGET_COMPLETED:
          String label = event.getId().getTargetCompleted().getLabel();
          String configMnemonic =
              configIdToMnemonic.get(event.getId().getTargetCompleted().getConfiguration().getId());
          event
              .getCompleted()
              .getOutputGroupList()
              .forEach(
                  o -> {
                    List<String> sets = getFileSets(o);
                    outputGroupToFileSets.putAll(o.getName(), sets);
                    targetToFileSets.putAll(label, sets);
                    sets.forEach(s -> fileSetConfigs.put(s, configMnemonic));
                  });
          continue;
        case STARTED:
          startTimeMillis = event.getStarted().getStartTimeMillis();
          continue;
        default: // continue
      }
    }

    Map<String, FileSet> filesMap = new LinkedHashMap<>();
    fileSets.forEach((id, files) -> filesMap.put(id, new FileSet(files, fileSetConfigs.get(id))));
    return new ParsedBepOutput(
        startTimeMillis, filesMap, outputGroupToFileSets.build(), targetToFileSets.build());
  }

  private static List<String> getFileSets(BuildEventStreamProtos.OutputGroup group) {
    return group.getFileSetsList().stream()
        .map(NamedSetOfFilesId::getId)
        .collect(Collectors.toList());
  }

  private static class FileSet {
    private final BuildEventStreamProtos.NamedSetOfFiles namedSet;
    private final String configuration;

    FileSet(BuildEventStreamProtos.NamedSetOfFiles namedSet, String configuration) {<commit_msg>Fix configuration handling in BEP parsing.<commit_after>/*
 * Copyright 2019 The Bazel Authors. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.google.idea.blaze.base.command.buildresult;

import static com.google.common.collect.ImmutableList.toImmutableList;
import static com.google.common.collect.ImmutableSet.toImmutableSet;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableListMultimap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.ImmutableSetMultimap;
import com.google.common.collect.Queues;
import com.google.common.collect.SetMultimap;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.BuildEventId.NamedSetOfFilesId;
import com.google.devtools.build.lib.buildeventstream.BuildEventStreamProtos.NamedSetOfFiles;
import com.google.idea.blaze.base.model.primitives.Label;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Queue;
import java.util.Set;
import java.util.function.Predicate;
import java.util.stream.Collectors;

/**
 * An intermediate data class representing blaze's build event protocol (BEP) output for a build.
 */
final class ParsedBepOutput {

  static ParsedBepOutput parseBepArtifacts(InputStream bepStream) throws IOException {
    BuildEventStreamProtos.BuildEvent event;
    Map<String, String> configIdToMnemonic = new HashMap<>();
    Map<String, NamedSetOfFiles> fileSets = new LinkedHashMap<>();
    Map<String, String> fileSetConfigs = new HashMap<>();
    ImmutableSetMultimap.Builder<String, String> outputGroupToFileSets =
        ImmutableSetMultimap.builder();
    ImmutableSetMultimap.Builder<String, String> targetToFileSets = ImmutableSetMultimap.builder();
    long startTimeMillis = 0L;

    while ((event = BuildEventStreamProtos.BuildEvent.parseDelimitedFrom(bepStream)) != null) {
      switch (event.getId().getIdCase()) {
        case CONFIGURATION:
          configIdToMnemonic.put(
              event.getId().getConfiguration().getId(), event.getConfiguration().getMnemonic());
          continue;
        case NAMED_SET:
          fileSets.put(event.getId().getNamedSet().getId(), event.getNamedSetOfFiles());
          continue;
        case TARGET_COMPLETED:
          String label = event.getId().getTargetCompleted().getLabel();
          String configMnemonic =
              configIdToMnemonic.get(event.getId().getTargetCompleted().getConfiguration().getId());
          event
              .getCompleted()
              .getOutputGroupList()
              .forEach(
                  o -> {
                    List<String> sets = getFileSets(o);
                    outputGroupToFileSets.putAll(o.getName(), sets);
                    targetToFileSets.putAll(label, sets);
                    sets.forEach(s -> fileSetConfigs.put(s, configMnemonic));
                  });
          continue;
        case STARTED:
          startTimeMillis = event.getStarted().getStartTimeMillis();
          continue;
        default: // continue
      }
    }
    fillInTransitiveFileSetConfigs(fileSets, fileSetConfigs);
    Map<String, FileSet> filesMap = new LinkedHashMap<>();
    fileSets.forEach((id, files) -> filesMap.put(id, new FileSet(files, fileSetConfigs.get(id))));
    return new ParsedBepOutput(
        startTimeMillis, filesMap, outputGroupToFileSets.build(), targetToFileSets.build());
  }

  private static List<String> getFileSets(BuildEventStreamProtos.OutputGroup group) {
    return group.getFileSetsList().stream()
        .map(NamedSetOfFilesId::getId)
        .collect(Collectors.toList());
  }

  /**
   * BEP explicitly lists the configuration mnemonics of top-level file sets. This method fills in a
   * file set ID to mnemonic map for the transitive closure.
   */
  private static void fillInTransitiveFileSetConfigs(
      Map<String, NamedSetOfFiles> fileSets, Map<String, String> fileSetConfigs) {
    Queue<String> toVisit = Queues.newArrayDeque();
    toVisit.addAll(fileSetConfigs.keySet());
    while (!toVisit.isEmpty()) {
      String setId = toVisit.remove();
      String config = fileSetConfigs.get(setId);
      fileSets.get(setId).getFileSetsList().stream()
          .map(NamedSetOfFilesId::getId)
          .filter(s -> !fileSetConfigs.containsKey(s))
          .forEach(
              child -> {
                fileSetConfigs.put(child, config);
                toVisit.add(child);
              });
    }
  }

  private static class FileSet {
    private final NamedSetOfFiles namedSet;
    private final String configuration;

    FileSet(NamedSetOfFiles namedSet, String configuration) {
{'fraction_non_alphanumeric': 0.1010973379394432, 'fraction_numerical': 0.001828896565738671, 'mean_word_length': 3.521359669269637, 'pattern_counts': {'":': 0, '<': 30, '<?xml version=': 0, '>': 38, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5454141', 'n_tokens_mistral': 2774, 'n_tokens_neox': 2746, 'n_words': 623}