<commit_before>// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.profiler.grapher;

import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.profiler.CpuUsageTimeSeries;
import com.google.gson.stream.JsonReader;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.StringJoiner;
import java.util.zip.GZIPInputStream;

/**
 * An experimental tool to turn Json profiles into graphs. Do not depend on the continuing existence
 * of this tool.
 *
 * <p>Run this tool like so: bazel run \
 * //src/main/java/com/google/devtools/build/lib/profiler:profiler-grapher \
 * -- /path/to/command.profile > /tmp/tmp.csv
 *
 * <p>Plot the resulting CSV with gnuplot like so: gnuplot -p -e \
 * "set datafile sep ','; plot for [col=1:3] '/tmp/tmp.csv' using col lw 2 with lines \
 * title columnheader"
 */
public class ProfileGrapher {
  public static final long DEFAULT_BUCKET_SIZE_MILLIS = 1000;

  // Decode a JSON object and flattens any nested JSON objects.
  private static Map<String, Object> decodeJsonObject(JsonReader reader) throws IOException {
    reader.beginObject();
    Map<String, Object> data = new HashMap<>();
    while (reader.hasNext()) {
      String name = reader.nextName();
      Object value;
      switch (reader.peek()) {
        case BOOLEAN:
          value = reader.nextBoolean();
          break;
        case NUMBER:
          value = reader.nextDouble();
          break;
        case STRING:
          value = reader.nextString();
          break;
        case BEGIN_OBJECT:
          value = null;
          Map<String, Object> childData = decodeJsonObject(reader);
          for (Map.Entry<String, Object> entry : childData.entrySet()) {
            data.put(name + "." + entry.getKey(), entry.getValue());
          }
          break;
        default:
          reader.skipValue();
          continue;
      }
      data.put(name, value);
    }
    reader.endObject();
    return data;
  }

  public static void main(String[] args) throws IOException {
    if (args.length != 1) {
      System.err.println("Requires filename of a Bazel profile in Chrome event trace format.");
      System.exit(1);
    }
    String filename = args[0];
    boolean gzipped = filename.endsWith(".gz");

    // TODO(twerth): Make it possible to select the set of profiler task descriptions on the command
    // line.
    ImmutableList<String> profilerTaskDescriptions =
        ImmutableList.of(
            "Remote execution process wall time",
            "action processing",
            "Remote execution file fetching");
    Map<String, CpuUsageTimeSeries> seriesMap = new LinkedHashMap<>();
    for (String profilerTaskDescription : profilerTaskDescriptions) {
      seriesMap.put(profilerTaskDescription, new CpuUsageTimeSeries(0, DEFAULT_BUCKET_SIZE_MILLIS));
    }

    long maxEndTime = 0;
    // TODO(twerth): Support gzip compressed profiles.
    try (JsonReader reader =
        new JsonReader(
            new BufferedReader(
                new InputStreamReader(
                    maybeUnzip(new FileInputStream(filename), gzipped), StandardCharsets.UTF_8)))) {
      reader.beginArray();
      while (reader.hasNext()) {
        Map<String, Object> data = decodeJsonObject(reader);
        Object name = data.get("name");
        if ("cpu counters".equals(name)) {
          seriesMap.putIfAbsent(
              "cpu counters", new CpuUsageTimeSeries(0, DEFAULT_BUCKET_SIZE_MILLIS));
          CpuUsageTimeSeries series = seriesMap.get(name);
          Double ts = (Double) data.get("ts");<commit_msg>ProfileGrapher: handle Json object profile<commit_after>// Copyright 2018 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.profiler.grapher;

import com.google.common.collect.ImmutableList;
import com.google.devtools.build.lib.profiler.CpuUsageTimeSeries;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonToken;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.StringJoiner;
import java.util.zip.GZIPInputStream;

/**
 * An experimental tool to turn Json profiles into graphs. Do not depend on the continuing existence
 * of this tool.
 *
 * <p>Run this tool like so: bazel run \
 * //src/main/java/com/google/devtools/build/lib/profiler:profiler-grapher \
 * -- /path/to/command.profile > /tmp/tmp.csv
 *
 * <p>Plot the resulting CSV with gnuplot like so: gnuplot -p -e \
 * "set datafile sep ','; plot for [col=1:3] '/tmp/tmp.csv' using col lw 2 with lines \
 * title columnheader"
 */
public class ProfileGrapher {
  public static final long DEFAULT_BUCKET_SIZE_MILLIS = 1000;

  // Decode a JSON object and flattens any nested JSON objects.
  private static Map<String, Object> decodeJsonObject(JsonReader reader) throws IOException {
    reader.beginObject();
    Map<String, Object> data = new HashMap<>();
    while (reader.hasNext()) {
      String name = reader.nextName();
      Object value;
      switch (reader.peek()) {
        case BOOLEAN:
          value = reader.nextBoolean();
          break;
        case NUMBER:
          value = reader.nextDouble();
          break;
        case STRING:
          value = reader.nextString();
          break;
        case BEGIN_OBJECT:
          value = null;
          Map<String, Object> childData = decodeJsonObject(reader);
          for (Map.Entry<String, Object> entry : childData.entrySet()) {
            data.put(name + "." + entry.getKey(), entry.getValue());
          }
          break;
        default:
          reader.skipValue();
          continue;
      }
      data.put(name, value);
    }
    reader.endObject();
    return data;
  }

  public static void main(String[] args) throws IOException {
    if (args.length != 1) {
      System.err.println("Requires filename of a Bazel profile in Chrome event trace format.");
      System.exit(1);
    }
    String filename = args[0];
    boolean gzipped = filename.endsWith(".gz");

    // TODO(twerth): Make it possible to select the set of profiler task descriptions on the command
    // line.
    ImmutableList<String> profilerTaskDescriptions =
        ImmutableList.of(
            "Remote execution process wall time",
            "action processing",
            "Remote execution file fetching");
    Map<String, CpuUsageTimeSeries> seriesMap = new LinkedHashMap<>();
    for (String profilerTaskDescription : profilerTaskDescriptions) {
      seriesMap.put(profilerTaskDescription, new CpuUsageTimeSeries(0, DEFAULT_BUCKET_SIZE_MILLIS));
    }

    long maxEndTime = 0;
    try (JsonReader reader =
        new JsonReader(
            new BufferedReader(
                new InputStreamReader(
                    maybeUnzip(new FileInputStream(filename), gzipped), StandardCharsets.UTF_8)))) {
      if (reader.peek() == JsonToken.BEGIN_OBJECT) {
        reader.beginObject();
        while (reader.hasNext()) {
          if ("traceEvents".equals(reader.nextName())) {
            break;
          }
          reader.skipValue();
        }
      }
      reader.beginArray();
      while (reader.hasNext()) {
        Map<String, Object> data = decodeJsonObject(reader);
        Object name = data.get("name");
        if ("cpu counters".equals(name)) {
          seriesMap.putIfAbsent(
              "cpu counters", new CpuUsageTimeSeries(0, DEFAULT_BUCKET_SIZE_MILLIS));
          CpuUsageTimeSeries series = seriesMap.get(name);
          Double ts = (Double) data.get("ts");
{'fraction_non_alphanumeric': 0.10668702956256136, 'fraction_numerical': 0.004799825460892331, 'mean_word_length': 2.8781725888324874, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5147628', 'n_tokens_mistral': 2594, 'n_tokens_neox': 2567, 'n_words': 762}