<commit_before>package edu.sdsc.scigraph.internal.reachability;

import static com.google.common.collect.Iterables.size;
import static java.lang.String.format;

import java.util.AbstractMap;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;

import org.neo4j.graphdb.Direction;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Path;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.traversal.TraversalDescription;
import org.neo4j.kernel.Traversal;
import org.neo4j.kernel.Uniqueness;
import org.neo4j.tooling.GlobalGraphOperations;

import com.google.common.collect.ImmutableSet;
import com.google.common.primitives.Longs;

public class ReachabilityIndex {

  private static final Logger logger = Logger.getLogger(ReachabilityIndex.class.getName());

  private static final String INDEX_EXISTS_PROPERTY = "ReachablilityIndexExists";
  private static final String IN_LIST_PROPERTY  = "ReachablilityIndexInList";
  private static final String OUT_LIST_PROPERTY = "ReachablilityIndexOutList";

  private final GraphDatabaseService graphDb;
  private final Set<Long> forbiddenNodes;
  private final Node metaDataNode;

  /***
   * Manage a reachability index object on a graph
   * @param graphDb The graph on which to build the reachability index
   */
  public ReachabilityIndex(GraphDatabaseService graphDb) {
    this(graphDb, Collections.<Long>emptySet());
  }

  /**
   * Manage a reachability index object on a graph.
   * @param graph The graph on which to build the reachability index
   * @param forbiddenNodes Nodes that should be not be traversed during index building
   */
  public ReachabilityIndex(GraphDatabaseService graphdb, Collection<Long> forbiddenNodes) {
    this.graphDb = graphdb;
    this.forbiddenNodes = ImmutableSet.copyOf(forbiddenNodes);
    metaDataNode = graphDb.getNodeById(0);
  }

  /***
   * @return if a reachability index has already been created on this graph.
   */
  public boolean indexExists() {
    return (boolean)metaDataNode.getProperty(INDEX_EXISTS_PROPERTY, false);
  }

  /**
   * Create a reachability index on a graph.
   */
  public void createIndex() {
    if (indexExists()) {
      throw new IllegalStateException("Reachability index already exists. Drop it first and then recreate it.");
    }

    long startTime = System.currentTimeMillis();

    Set<Entry<Long, Integer>> hopCoverages = getHopCoverages();

    long endTime = System.currentTimeMillis();

    logger.fine(format("Takes %d second(s) to calculate HopCoverage",
        TimeUnit.MILLISECONDS.toSeconds(endTime - startTime)));

    MemoryReachabilityIndex inMemoryIndex = new MemoryReachabilityIndex();

    TraversalDescription incomingTraversal = Traversal.description().breadthFirst().uniqueness(Uniqueness.NODE_GLOBAL)
        .expand(new DirectionalPathExpander(Direction.INCOMING))
        .evaluator(new ReachabilityEvaluator(inMemoryIndex, Direction.INCOMING, forbiddenNodes));

    TraversalDescription outgoingTraversal = Traversal.description().breadthFirst().uniqueness(Uniqueness.NODE_GLOBAL)
        .expand(new DirectionalPathExpander(Direction.OUTGOING))
        .evaluator(new ReachabilityEvaluator(inMemoryIndex, Direction.OUTGOING, forbiddenNodes));

    long bfsTime = 0, rbfsTime = 0;
    for (Entry<Long, Integer> coverage : hopCoverages) {
      Node workingNode = graphDb.getNodeById(coverage.getKey());
      startTime = System.currentTimeMillis();
      for (Path p: incomingTraversal.traverse(workingNode)) {
        logger.finest(p.toString()); // Avoids unused variable warning
      }
      endTime = System.currentTimeMillis() ;
      rbfsTime  += endTime - startTime;
      for (Path p :outgoingTraversal.traverse(workingNode)) {
        logger.finest(p.toString()); // Avoids unused variable warning 
      }
      bfsTime += System.currentTimeMillis() - endTime;
    }

    logger.fine("BFS index building time: " + (bfsTime/1000) + " sec(s), RBFS index building time: " + (rbfsTime /1000) + " sec(s).");
    // store the inMemoryIdx into db.
    Transaction tx = graphDb.beginTx();

    int operationCount = 0;
    System.out.println(inMemoryIndex);
    for(Entry<Long, InOutList> e: inMemoryIndex.entrySet() ) {
      Node node = graphDb.getNodeById(e.getKey());
      operationCount++;
      if ( operationCount % 500000 == 0 ) {
        logger.fine("commit transaction when populating in-out list.");
        tx.success();
        tx.finish();
        tx = graphDb.beginTx();
      }
      node.setProperty(IN_LIST_PROPERTY, Longs.toArray(e.getValue().getInList()));
      node.setProperty(OUT_LIST_PROPERTY, Longs.toArray(e.getValue().getOutList()));
    }<commit_msg>Removes println<commit_after>package edu.sdsc.scigraph.internal.reachability;

import static com.google.common.collect.Iterables.size;
import static java.lang.String.format;

import java.util.AbstractMap;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import java.util.logging.Logger;

import org.neo4j.graphdb.Direction;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Path;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.traversal.TraversalDescription;
import org.neo4j.kernel.Traversal;
import org.neo4j.kernel.Uniqueness;
import org.neo4j.tooling.GlobalGraphOperations;

import com.google.common.collect.ImmutableSet;
import com.google.common.primitives.Longs;

public class ReachabilityIndex {

  private static final Logger logger = Logger.getLogger(ReachabilityIndex.class.getName());

  private static final String INDEX_EXISTS_PROPERTY = "ReachablilityIndexExists";
  private static final String IN_LIST_PROPERTY  = "ReachablilityIndexInList";
  private static final String OUT_LIST_PROPERTY = "ReachablilityIndexOutList";

  private final GraphDatabaseService graphDb;
  private final Set<Long> forbiddenNodes;
  private final Node metaDataNode;

  /***
   * Manage a reachability index object on a graph
   * @param graphDb The graph on which to build the reachability index
   */
  public ReachabilityIndex(GraphDatabaseService graphDb) {
    this(graphDb, Collections.<Long>emptySet());
  }

  /**
   * Manage a reachability index object on a graph.
   * @param graph The graph on which to build the reachability index
   * @param forbiddenNodes Nodes that should be not be traversed during index building
   */
  public ReachabilityIndex(GraphDatabaseService graphdb, Collection<Long> forbiddenNodes) {
    this.graphDb = graphdb;
    this.forbiddenNodes = ImmutableSet.copyOf(forbiddenNodes);
    metaDataNode = graphDb.getNodeById(0);
  }

  /***
   * @return if a reachability index has already been created on this graph.
   */
  public boolean indexExists() {
    return (boolean)metaDataNode.getProperty(INDEX_EXISTS_PROPERTY, false);
  }

  /**
   * Create a reachability index on a graph.
   */
  public void createIndex() {
    if (indexExists()) {
      throw new IllegalStateException("Reachability index already exists. Drop it first and then recreate it.");
    }

    long startTime = System.currentTimeMillis();

    Set<Entry<Long, Integer>> hopCoverages = getHopCoverages();

    long endTime = System.currentTimeMillis();

    logger.fine(format("Takes %d second(s) to calculate HopCoverage",
        TimeUnit.MILLISECONDS.toSeconds(endTime - startTime)));

    MemoryReachabilityIndex inMemoryIndex = new MemoryReachabilityIndex();

    TraversalDescription incomingTraversal = Traversal.description().breadthFirst().uniqueness(Uniqueness.NODE_GLOBAL)
        .expand(new DirectionalPathExpander(Direction.INCOMING))
        .evaluator(new ReachabilityEvaluator(inMemoryIndex, Direction.INCOMING, forbiddenNodes));

    TraversalDescription outgoingTraversal = Traversal.description().breadthFirst().uniqueness(Uniqueness.NODE_GLOBAL)
        .expand(new DirectionalPathExpander(Direction.OUTGOING))
        .evaluator(new ReachabilityEvaluator(inMemoryIndex, Direction.OUTGOING, forbiddenNodes));

    long bfsTime = 0, rbfsTime = 0;
    for (Entry<Long, Integer> coverage : hopCoverages) {
      Node workingNode = graphDb.getNodeById(coverage.getKey());
      startTime = System.currentTimeMillis();
      for (Path p: incomingTraversal.traverse(workingNode)) {
        logger.finest(p.toString()); // Avoids unused variable warning
      }
      endTime = System.currentTimeMillis() ;
      rbfsTime  += endTime - startTime;
      for (Path p :outgoingTraversal.traverse(workingNode)) {
        logger.finest(p.toString()); // Avoids unused variable warning 
      }
      bfsTime += System.currentTimeMillis() - endTime;
    }

    logger.fine("BFS index building time: " + (bfsTime/1000) + " sec(s), RBFS index building time: " + (rbfsTime /1000) + " sec(s).");
    // store the inMemoryIdx into db.
    Transaction tx = graphDb.beginTx();

    int operationCount = 0;
    for(Entry<Long, InOutList> e: inMemoryIndex.entrySet() ) {
      Node node = graphDb.getNodeById(e.getKey());
      operationCount++;
      if ( operationCount % 500000 == 0 ) {
        logger.fine("commit transaction when populating in-out list.");
        tx.success();
        tx.finish();
        tx = graphDb.beginTx();
      }
      node.setProperty(IN_LIST_PROPERTY, Longs.toArray(e.getValue().getInList()));
      node.setProperty(OUT_LIST_PROPERTY, Longs.toArray(e.getValue().getOutList()));
    }
{'fraction_non_alphanumeric': 0.10377262038646355, 'fraction_numerical': 0.00572538595235661, 'mean_word_length': 4.205960617349654, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5046554', 'n_tokens_mistral': 2964, 'n_tokens_neox': 2880, 'n_words': 660}