<commit_before>/*
 * Copyright 2014 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.devtools.kythe.analyzers.base;

import com.google.common.base.Preconditions;
import com.google.devtools.kythe.common.FormattingLogger;
import com.google.devtools.kythe.platform.shared.StatisticsCollector;
import com.google.devtools.kythe.proto.Analysis.CompilationUnit.FileInput;
import com.google.devtools.kythe.proto.MarkedSource;
import com.google.devtools.kythe.proto.MarkedSource.Kind;
import com.google.devtools.kythe.proto.Storage.VName;
import com.google.devtools.kythe.util.KytheURI;
import com.google.devtools.kythe.util.Span;
import java.nio.charset.Charset;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import javax.annotation.Nullable;

/**
 * Factory for Kythe-compliant node and edge {@link EntrySet}s. In general, this class provides two
 * sets of methods: low-level node/edge builders and higher-level abstractions for specific kinds of
 * nodes/edges such as ANCHOR nodes and their edges. Each higher-level abstraction returns a fully
 * realized {@link EntrySet} and automatically emits it (and any associated edges). The lower-level
 * methods, such as {@link #newNode(String)} return {@link EntrySet.Builder}s and must be emitted by
 * the client.
 *
 * <p>This class is meant to be subclassed to build indexer-specific nodes and edges.
 */
public class KytheEntrySets {
  private static final FormattingLogger logger = FormattingLogger.getLogger(KytheEntrySets.class);

  public static final String NODE_PREFIX = "/kythe/";

  private final Map<String, EntrySet> nameNodes = new HashMap<>();

  private final StatisticsCollector statistics;
  private final FactEmitter emitter;
  private final String language;

  private final VName compilationVName;
  private final Map<String, VName> inputVNames;

  public KytheEntrySets(
      StatisticsCollector statistics,
      FactEmitter emitter,
      VName compilationVName,
      List<FileInput> requiredInputs) {
    this.statistics = statistics;
    this.emitter = emitter;
    this.language = compilationVName.getLanguage();
    this.compilationVName = compilationVName;

    inputVNames = new HashMap<>();
    for (FileInput input : requiredInputs) {
      String digest = input.getInfo().getDigest();
      VName.Builder name = input.getVName().toBuilder();
      Preconditions.checkArgument(
          !name.getPath().isEmpty(), "Required input VName must have non-empty path");
      if (name.getSignature().isEmpty()) {
        // Ensure file VName has digest signature
        name = name.setSignature(digest);
      }
      if (inputVNames.containsKey(digest)) {
        statistics.incrementCounter("file-digest-collision");
        logger.warningfmt(
            "Found two files with the same digest [%s]: %s and %s",
            digest, name, inputVNames.get(digest));
      }
      inputVNames.put(digest, name.build());
    }
  }

  /** Returns the {@link FactEmitter} used to emit generated {@link EntrySet}s. */
  public final FactEmitter getEmitter() {
    return emitter;
  }

  /** Return the {@link StatisticsCollector} being used. */
  protected final StatisticsCollector getStatisticsCollector() {
    return statistics;
  }

  /**
   * Returns a new {@link NodeBuilder} with the given node kind set.
   *
   * @deprecated use {@link #newNode(NodeKind)} for schema-defined kinds
   */
  @Deprecated
  public NodeBuilder newNode(String kind) {
    getStatisticsCollector().incrementCounter("deprecated-new-node-" + kind);
    return new NodeBuilder(kind, null, language);
  }

  /** Returns a new {@link NodeBuilder} with the given node kind set. */
  public NodeBuilder newNode(NodeKind kind) {
    getStatisticsCollector().incrementCounter("new-node-" + kind);
    return new NodeBuilder(kind, language);
  }

  /** Returns (and emits) a new builtin node. */
  public EntrySet newBuiltinAndEmit(String name) {
    return newBuiltinAndEmit(name, Optional.empty());
  }

  /** Returns (and emits) a new builtin node. */
  public EntrySet newBuiltinAndEmit(String name, Optional<String> docUri) {
    EntrySet.Builder nodeBuilder =
        newNode(NodeKind.TBUILTIN)
            .setSignature(getBuiltinSignature(name))
            .setProperty(
                "code",
                MarkedSource.newBuilder()
                    .setPreText(name)
                    .setKind(Kind.IDENTIFIER)
                    .build());
    if (docUri.isPresent()) {
      setDocumentUriProperty(nodeBuilder, docUri.get());
    }
    return emitAndReturn(nodeBuilder);
  }

  /** Returns a VName for the builtin node corresponding to the specified name. */
  public VName getBuiltinVName(String name) {
    return VName.newBuilder()
        .setSignature(getBuiltinSignature(name))
        .setLanguage(this.language)
        .build();
  }
  /**
   * Returns (and emits) a new anchor node at the given location in the file with an optional
   * snippet span.
   */
  public EntrySet newAnchorAndEmit(VName fileVName, Span loc, Span snippet) {
    if (loc == null || !loc.valid()) {
      // TODO(schroederc): reduce number of invalid anchors<commit_msg>Java indexer: fix potential NullPointerException<commit_after>/*
 * Copyright 2014 Google Inc. All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.google.devtools.kythe.analyzers.base;

import com.google.common.base.Preconditions;
import com.google.devtools.kythe.common.FormattingLogger;
import com.google.devtools.kythe.platform.shared.StatisticsCollector;
import com.google.devtools.kythe.proto.Analysis.CompilationUnit.FileInput;
import com.google.devtools.kythe.proto.MarkedSource;
import com.google.devtools.kythe.proto.MarkedSource.Kind;
import com.google.devtools.kythe.proto.Storage.VName;
import com.google.devtools.kythe.util.KytheURI;
import com.google.devtools.kythe.util.Span;
import java.nio.charset.Charset;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import javax.annotation.Nullable;

/**
 * Factory for Kythe-compliant node and edge {@link EntrySet}s. In general, this class provides two
 * sets of methods: low-level node/edge builders and higher-level abstractions for specific kinds of
 * nodes/edges such as ANCHOR nodes and their edges. Each higher-level abstraction returns a fully
 * realized {@link EntrySet} and automatically emits it (and any associated edges). The lower-level
 * methods, such as {@link #newNode(String)} return {@link EntrySet.Builder}s and must be emitted by
 * the client.
 *
 * <p>This class is meant to be subclassed to build indexer-specific nodes and edges.
 */
public class KytheEntrySets {
  private static final FormattingLogger logger = FormattingLogger.getLogger(KytheEntrySets.class);

  public static final String NODE_PREFIX = "/kythe/";

  private final Map<String, EntrySet> nameNodes = new HashMap<>();

  private final StatisticsCollector statistics;
  private final FactEmitter emitter;
  private final String language;

  private final VName compilationVName;
  private final Map<String, VName> inputVNames;

  public KytheEntrySets(
      StatisticsCollector statistics,
      FactEmitter emitter,
      VName compilationVName,
      List<FileInput> requiredInputs) {
    this.statistics = statistics;
    this.emitter = emitter;
    this.language = compilationVName.getLanguage();
    this.compilationVName = compilationVName;

    inputVNames = new HashMap<>();
    for (FileInput input : requiredInputs) {
      String digest = input.getInfo().getDigest();
      VName.Builder name = input.getVName().toBuilder();
      Preconditions.checkArgument(
          !name.getPath().isEmpty(), "Required input VName must have non-empty path");
      if (name.getSignature().isEmpty()) {
        // Ensure file VName has digest signature
        name = name.setSignature(digest);
      }
      if (inputVNames.containsKey(digest)) {
        statistics.incrementCounter("file-digest-collision");
        logger.warningfmt(
            "Found two files with the same digest [%s]: %s and %s",
            digest, name, inputVNames.get(digest));
      }
      inputVNames.put(digest, name.build());
    }
  }

  /** Returns the {@link FactEmitter} used to emit generated {@link EntrySet}s. */
  public final FactEmitter getEmitter() {
    return emitter;
  }

  /** Return the {@link StatisticsCollector} being used. */
  protected final StatisticsCollector getStatisticsCollector() {
    return statistics;
  }

  /**
   * Returns a new {@link NodeBuilder} with the given node kind set.
   *
   * @deprecated use {@link #newNode(NodeKind)} for schema-defined kinds
   */
  @Deprecated
  public NodeBuilder newNode(String kind) {
    getStatisticsCollector().incrementCounter("deprecated-new-node-" + kind);
    return new NodeBuilder(kind, Optional.empty(), language);
  }

  /** Returns a new {@link NodeBuilder} with the given node kind set. */
  public NodeBuilder newNode(NodeKind kind) {
    getStatisticsCollector().incrementCounter("new-node-" + kind);
    return new NodeBuilder(kind, language);
  }

  /** Returns (and emits) a new builtin node. */
  public EntrySet newBuiltinAndEmit(String name) {
    return newBuiltinAndEmit(name, Optional.empty());
  }

  /** Returns (and emits) a new builtin node. */
  public EntrySet newBuiltinAndEmit(String name, Optional<String> docUri) {
    EntrySet.Builder nodeBuilder =
        newNode(NodeKind.TBUILTIN)
            .setSignature(getBuiltinSignature(name))
            .setProperty(
                "code",
                MarkedSource.newBuilder().setPreText(name).setKind(Kind.IDENTIFIER).build());
    if (docUri.isPresent()) {
      setDocumentUriProperty(nodeBuilder, docUri.get());
    }
    return emitAndReturn(nodeBuilder);
  }

  /** Returns a VName for the builtin node corresponding to the specified name. */
  public VName getBuiltinVName(String name) {
    return VName.newBuilder()
        .setSignature(getBuiltinSignature(name))
        .setLanguage(this.language)
        .build();
  }
  /**
   * Returns (and emits) a new anchor node at the given location in the file with an optional
   * snippet span.
   */
  public EntrySet newAnchorAndEmit(VName fileVName, Span loc, Span snippet) {
    if (loc == null || !loc.valid()) {
      // TODO(schroederc): reduce number of invalid anchors
{'fraction_non_alphanumeric': 0.1002246414377052, 'fraction_numerical': 0.0013824088474166235, 'mean_word_length': 3.6207584830339323, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3437730', 'n_tokens_mistral': 3271, 'n_tokens_neox': 3205, 'n_words': 994}