<commit_before>/**
 * Copyright (C) 2014 The SciGraph authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package edu.sdsc.scigraph.owlapi;

import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Logger;

import javax.inject.Named;

import org.neo4j.cypher.javacompat.ExecutionEngine;
import org.neo4j.graphdb.Direction;
import org.neo4j.graphdb.DynamicLabel;
import org.neo4j.graphdb.DynamicRelationshipType;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Label;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Path;
import org.neo4j.graphdb.Relationship;
import org.neo4j.graphdb.RelationshipType;
import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.index.ReadableIndex;
import org.neo4j.graphdb.traversal.Uniqueness;

import edu.sdsc.scigraph.frames.CommonProperties;
import edu.sdsc.scigraph.frames.Concept;
import edu.sdsc.scigraph.neo4j.GraphUtil;

public class OwlPostprocessor {

  private static final Logger logger = Logger.getLogger(OwlPostprocessor.class.getName());

  private final GraphDatabaseService graphDb;
  private final ExecutionEngine engine;
  private final ReadableIndex<Node> nodeIndex;

  private final Map<String, String> categoryMap;

  public OwlPostprocessor(GraphDatabaseService graphDb,
      @Named("owl.categories") Map<String, String> categoryMap) {
    this.graphDb = graphDb;
    this.categoryMap = categoryMap;
    this.nodeIndex = graphDb.index().getNodeAutoIndexer().getAutoIndex();
    engine = new ExecutionEngine(graphDb);
  }

  public void postprocess() {
    processSomeValuesFrom();
    processCategories(categoryMap);
  }

  public void processSomeValuesFrom() {
    logger.info("Processing someValuesFrom classes");
    try (Transaction tx = graphDb.beginTx()) {
      ResourceIterator<Map<String, Object>> results =
          engine.execute(
              "MATCH (n)-[relationship]->(svf:someValuesFrom)-[:property]->(p) "
                  + "RETURN n, relationship, svf, p").iterator();
      while (results.hasNext()) {
        Map<String, Object> result = results.next();
        Node subject = (Node) result.get("n");
        Relationship relationship = (Relationship) result.get("relationship");
        Node svf = (Node) result.get("svf");
        Node property = (Node) result.get("p");
        for (Relationship r : svf.getRelationships(OwlRelationships.FILLER)) {
          Node object = r.getEndNode();
          String relationshipName =
              GraphUtil.getProperty(property, CommonProperties.FRAGMENT, String.class)
              .or(GraphUtil.getProperty(property, CommonProperties.URI, String.class).get());
          RelationshipType type = DynamicRelationshipType.withName(relationshipName);
          String propertyUri =
              GraphUtil.getProperty(property, CommonProperties.URI, String.class).get();
          Relationship inferred = subject.createRelationshipTo(object, type);
          inferred.setProperty(CommonProperties.URI, propertyUri);
          inferred.setProperty(CommonProperties.CONVENIENCE, true);
          inferred.setProperty(CommonProperties.OWL_TYPE, relationship.getType().name());
        }
      }
      tx.success();
    }
  }

  void processCategory(Node root, RelationshipType type, Direction direction, String category) {
    logger.info("Processing category: " + category);
    int count = 0;
    int batchSize = 10_000;
    Label label = DynamicLabel.label(category);
    Transaction tx = graphDb.beginTx();
    for (Path position : graphDb.traversalDescription().uniqueness(Uniqueness.NODE_GLOBAL)
        .depthFirst().relationships(type, direction)
        .relationships(OwlRelationships.RDF_TYPE, Direction.INCOMING)
        .relationships(OwlRelationships.OWL_EQUIVALENT_CLASS, Direction.BOTH).traverse(root)) {
      Node end = position.endNode();
      GraphUtil.addProperty(end, Concept.CATEGORY, category);
      end.addLabel(label);
      if (0 == ++count % batchSize) {
        logger.fine("Commiting " + count);
        tx.success();
        tx.close();
        tx = graphDb.beginTx();
      }
    }
    tx.success();
    tx.close();
  }

  public void processCategories(Map<String, String> categories) {
    logger.info("Processing categories");
    for (Entry<String, String> category : categories.entrySet()) {
      Node root = null;
      try (Transaction tx = graphDb.beginTx()) {
        root = nodeIndex.get(CommonProperties.URI, category.getKey()).getSingle();
        tx.success();
      }
      if (null == root) {
        logger.warning("Failed to locate " + category.getKey() + " while processing categories");
      } else {
        processCategory(root, OwlRelationships.RDFS_SUBCLASS_OF, Direction.INCOMING,
            category.getValue());
      }
    }
  }

}<commit_msg>Logs number of categories processed.<commit_after>/**
 * Copyright (C) 2014 The SciGraph authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package edu.sdsc.scigraph.owlapi;

import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Logger;

import javax.inject.Named;

import org.neo4j.cypher.javacompat.ExecutionEngine;
import org.neo4j.graphdb.Direction;
import org.neo4j.graphdb.DynamicLabel;
import org.neo4j.graphdb.DynamicRelationshipType;
import org.neo4j.graphdb.GraphDatabaseService;
import org.neo4j.graphdb.Label;
import org.neo4j.graphdb.Node;
import org.neo4j.graphdb.Path;
import org.neo4j.graphdb.Relationship;
import org.neo4j.graphdb.RelationshipType;
import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.graphdb.Transaction;
import org.neo4j.graphdb.index.ReadableIndex;
import org.neo4j.graphdb.traversal.Uniqueness;

import edu.sdsc.scigraph.frames.CommonProperties;
import edu.sdsc.scigraph.frames.Concept;
import edu.sdsc.scigraph.neo4j.GraphUtil;

public class OwlPostprocessor {

  private static final Logger logger = Logger.getLogger(OwlPostprocessor.class.getName());

  private final GraphDatabaseService graphDb;
  private final ExecutionEngine engine;
  private final ReadableIndex<Node> nodeIndex;

  private final Map<String, String> categoryMap;

  public OwlPostprocessor(GraphDatabaseService graphDb,
      @Named("owl.categories") Map<String, String> categoryMap) {
    this.graphDb = graphDb;
    this.categoryMap = categoryMap;
    this.nodeIndex = graphDb.index().getNodeAutoIndexer().getAutoIndex();
    engine = new ExecutionEngine(graphDb);
  }

  public void postprocess() {
    processSomeValuesFrom();
    processCategories(categoryMap);
  }

  public void processSomeValuesFrom() {
    logger.info("Processing someValuesFrom classes");
    try (Transaction tx = graphDb.beginTx()) {
      ResourceIterator<Map<String, Object>> results =
          engine.execute(
              "MATCH (n)-[relationship]->(svf:someValuesFrom)-[:property]->(p) "
                  + "RETURN n, relationship, svf, p").iterator();
      while (results.hasNext()) {
        Map<String, Object> result = results.next();
        Node subject = (Node) result.get("n");
        Relationship relationship = (Relationship) result.get("relationship");
        Node svf = (Node) result.get("svf");
        Node property = (Node) result.get("p");
        for (Relationship r : svf.getRelationships(OwlRelationships.FILLER)) {
          Node object = r.getEndNode();
          String relationshipName =
              GraphUtil.getProperty(property, CommonProperties.FRAGMENT, String.class)
              .or(GraphUtil.getProperty(property, CommonProperties.URI, String.class).get());
          RelationshipType type = DynamicRelationshipType.withName(relationshipName);
          String propertyUri =
              GraphUtil.getProperty(property, CommonProperties.URI, String.class).get();
          Relationship inferred = subject.createRelationshipTo(object, type);
          inferred.setProperty(CommonProperties.URI, propertyUri);
          inferred.setProperty(CommonProperties.CONVENIENCE, true);
          inferred.setProperty(CommonProperties.OWL_TYPE, relationship.getType().name());
        }
      }
      tx.success();
    }
  }

  long processCategory(Node root, RelationshipType type, Direction direction, String category) {
    long count = 0;
    int batchSize = 10_000;
    Label label = DynamicLabel.label(category);
    Transaction tx = graphDb.beginTx();
    for (Path position : graphDb.traversalDescription().uniqueness(Uniqueness.NODE_GLOBAL)
        .depthFirst().relationships(type, direction)
        .relationships(OwlRelationships.RDF_TYPE, Direction.INCOMING)
        .relationships(OwlRelationships.OWL_EQUIVALENT_CLASS, Direction.BOTH).traverse(root)) {
      Node end = position.endNode();
      GraphUtil.addProperty(end, Concept.CATEGORY, category);
      end.addLabel(label);
      if (0 == ++count % batchSize) {
        logger.fine("Commiting " + count);
        tx.success();
        tx.close();
        tx = graphDb.beginTx();
      }
    }
    tx.success();
    tx.close();
    return count;
  }

  public void processCategories(Map<String, String> categories) {
    logger.info("Processing categories");
    for (Entry<String, String> category : categories.entrySet()) {
      Node root = null;
      try (Transaction tx = graphDb.beginTx()) {
        root = nodeIndex.get(CommonProperties.URI, category.getKey()).getSingle();
        tx.success();
      }
      if (null == root) {
        logger.warning("Failed to locate " + category.getKey() + " while processing categories");
      } else {
        logger.info("Processing category: " + category);
        long count = processCategory(root, OwlRelationships.RDFS_SUBCLASS_OF, Direction.INCOMING,
            category.getValue());
        logger.info("Processsed " + count + " nodes for " + category);
      }
    }
  }

}
{'fraction_non_alphanumeric': 0.11193202807536018, 'fraction_numerical': 0.005541189508681197, 'mean_word_length': 3.5634218289085546, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7323032', 'n_tokens_mistral': 3141, 'n_tokens_neox': 3059, 'n_words': 701}