<commit_before>/*
 *
 *  *  Copyright 2014 Orient Technologies LTD (info(at)orientechnologies.com)
 *  *
 *  *  Licensed under the Apache License, Version 2.0 (the "License");
 *  *  you may not use this file except in compliance with the License.
 *  *  You may obtain a copy of the License at
 *  *
 *  *       http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *  Unless required by applicable law or agreed to in writing, software
 *  *  distributed under the License is distributed on an "AS IS" BASIS,
 *  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *  See the License for the specific language governing permissions and
 *  *  limitations under the License.
 *  *
 *  * For more information: http://www.orientechnologies.com
 *  
 */

package com.orientechnologies.orient.graph.console;

import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.impls.orient.OrientBaseGraph;
import com.tinkerpop.blueprints.impls.orient.OrientVertex;
import com.tinkerpop.blueprints.util.io.graphml.GraphMLTokens;

import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.stream.events.XMLEvent;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;

/**
 * GraphMLReader writes the data from a GraphML stream to a graph. Derived from Blueprints GraphMLReader. Supports also vertex
 * labels.
 *
 * @author <NAME> (l.garulli(at)orientechnologies.com)
 * @author <NAME> (http://markorodriguez.com)
 * @author <NAME> (<EMAIL>)
 * @author <NAME> (http://fortytwo.net)
 */
public class OGraphMLReader {
  private static final String   LABELS           = "labels";
  private final OrientBaseGraph graph;
  private int                   vertexLabelIndex = 0;
  private String                vertexIdKey      = null;
  private String                edgeIdKey        = null;
  private String                edgeLabelKey     = null;

  /**
   * @param graph
   *          the graph to populate with the GraphML data
   */
  public OGraphMLReader(OrientBaseGraph graph) {
    this.graph = graph;
  }

  /**
   * Input the GraphML stream data into the graph. In practice, usually the provided graph is empty.
   *
   * @param inputGraph
   *          the graph to populate with the GraphML data
   * @param graphMLInputStream
   *          an InputStream of GraphML data
   * @throws IOException
   *           thrown when the GraphML data is not correctly formatted
   */
  public void inputGraph(final Graph inputGraph, final InputStream graphMLInputStream) throws IOException {
    inputGraph(inputGraph, graphMLInputStream, 1000, null, null, null);
  }

  /**
   * Input the GraphML stream data into the graph. In practice, usually the provided graph is empty.
   *
   * @param inputGraph
   *          the graph to populate with the GraphML data
   * @param filename
   *          name of a file containing GraphML data
   * @throws IOException
   *           thrown when the GraphML data is not correctly formatted
   */
  public void inputGraph(final Graph inputGraph, final String filename) throws IOException {
    inputGraph(inputGraph, filename, 1000, null, null, null);
  }

  /**
   * Input the GraphML stream data into the graph. More control over how data is streamed is provided by this method.
   *
   * @param inputGraph
   *          the graph to populate with the GraphML data
   * @param filename
   *          name of a file containing GraphML data
   * @param bufferSize
   *          the amount of elements to hold in memory before committing a transactions (only valid for TransactionalGraphs)
   * @param vertexIdKey
   *          if the id of a vertex is a &lt;data/&gt; property, fetch it from the data property.
   * @param edgeIdKey
   *          if the id of an edge is a &lt;data/&gt; property, fetch it from the data property.
   * @param edgeLabelKey
   *          if the label of an edge is a &lt;data/&gt; property, fetch it from the data property.
   * @throws IOException
   *           thrown when the GraphML data is not correctly formatted
   */
  public void inputGraph(final Graph inputGraph, final String filename, int bufferSize, String vertexIdKey, String edgeIdKey,
      String edgeLabelKey) throws IOException {
    FileInputStream fis = new FileInputStream(filename);
    try {
      inputGraph(inputGraph, fis, bufferSize, vertexIdKey, edgeIdKey, edgeLabelKey);
    } finally {
      fis.close();
    }
  }

  /**
   * Input the GraphML stream data into the graph. More control over how data is streamed is provided by this method.
   *
   * @param inputGraph
   *          the graph to populate with the GraphML data
   * @param graphMLInputStream
   *          an InputStream of GraphML data
   * @param bufferSize
   *          the amount of elements to hold in memory before committing a transactions (only valid for TransactionalGraphs)
   * @param vertexIdKey
   *          if the id of a vertex is a &lt;data/&gt; property, fetch it from the data property.
   * @param edgeIdKey
   *          if the id of an edge is a &lt;data/&gt; property, fetch it from the data property.
   * @param edgeLabelKey
   *          if the label of an edge is a &lt;data/&gt; property, fetch it from the data property.
   * @throws IOException
   *           thrown when the GraphML data is not correctly formatted
   */
  public void inputGraph(final Graph inputGraph, final InputStream graphMLInputStream, int bufferSize, String vertexIdKey,
      String edgeIdKey, String edgeLabelKey) throws IOException {

    XMLInputFactory inputFactory = XMLInputFactory.newInstance();

    try {
      XMLStreamReader reader = inputFactory.createXMLStreamReader(graphMLInputStream);

      final OrientBaseGraph graph = (OrientBaseGraph) inputGraph;

      Map<String, String> keyIdMap = new HashMap<String, String>();
      Map<String, String> keyTypesMaps = new HashMap<String, String>();
      // <Mapped ID String, ID Object>

      // <Default ID String, Mapped ID String>
      Map<String, String> vertexMappedIdMap = new HashMap<String, String>();

      // Buffered Vertex Data
      String vertexId = null;
      Map<String, Object> vertexProps = null;
      boolean inVertex = false;

      // Buffered Edge Data
      String edgeId = null;
      String edgeLabel = null;
      String vertexLabel = null;
      Vertex[] edgeEndVertices = null; // [0] = outVertex , [1] = inVertex
      Map<String, Object> edgeProps = null;
      boolean inEdge = false;

      int bufferCounter = 0;

      while (reader.hasNext()) {

        Integer eventType = reader.next();
        if (eventType.equals(XMLEvent.START_ELEMENT)) {
          String elementName = reader.getName().getLocalPart();

          if (elementName.equals(GraphMLTokens.KEY)) {
            String id = reader.getAttributeValue(null, GraphMLTokens.ID);
            String attributeName = reader.getAttributeValue(null, GraphMLTokens.ATTR_NAME);
            String attributeType = reader.getAttributeValue(null, GraphMLTokens.ATTR_TYPE);
            keyIdMap.put(id, attributeName);
            keyTypesMaps.put(id, attributeType);

          } else if (elementName.equals(GraphMLTokens.NODE)) {
            vertexId = reader.getAttributeValue(null, GraphMLTokens.ID);
            vertexLabel = reader.getAttributeValue(null, LABELS);
            if (vertexLabel != null) {
              if (vertexLabel.startsWith(":"))
                // REMOVE : AS PREFIX
                vertexLabel = vertexLabel.substring(1);

              final String[] vertexLabels = vertexLabel.split(",");

              // GET ONLY FIRST LABEL AS CLASS
              vertexLabel = "class:" + vertexLabels[vertexLabelIndex];
            }
            if (vertexIdKey != null)
              vertexMappedIdMap.put(vertexId, vertexId);
            inVertex = true;<commit_msg>Fixed multi-label separator in #2870<commit_after>/*
 *
 *  *  Copyright 2014 Orient Technologies LTD (info(at)orientechnologies.com)
 *  *
 *  *  Licensed under the Apache License, Version 2.0 (the "License");
 *  *  you may not use this file except in compliance with the License.
 *  *  You may obtain a copy of the License at
 *  *
 *  *       http://www.apache.org/licenses/LICENSE-2.0
 *  *
 *  *  Unless required by applicable law or agreed to in writing, software
 *  *  distributed under the License is distributed on an "AS IS" BASIS,
 *  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  *  See the License for the specific language governing permissions and
 *  *  limitations under the License.
 *  *
 *  * For more information: http://www.orientechnologies.com
 *  
 */

package com.orientechnologies.orient.graph.console;

import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Graph;
import com.tinkerpop.blueprints.Vertex;
import com.tinkerpop.blueprints.impls.orient.OrientBaseGraph;
import com.tinkerpop.blueprints.impls.orient.OrientVertex;
import com.tinkerpop.blueprints.util.io.graphml.GraphMLTokens;

import javax.xml.stream.XMLInputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
import javax.xml.stream.events.XMLEvent;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.Map;

/**
 * GraphMLReader writes the data from a GraphML stream to a graph. Derived from Blueprints GraphMLReader. Supports also vertex
 * labels.
 *
 * @author <NAME> (l.garulli(at)orientechnologies.com)
 * @author <NAME> (http://markorodriguez.com)
 * @author <NAME> (<EMAIL>)
 * @author <NAME> (http://fortytwo.net)
 */
public class OGraphMLReader {
  private static final String   LABELS           = "labels";
  private final OrientBaseGraph graph;
  private int                   vertexLabelIndex = 0;
  private String                vertexIdKey      = null;
  private String                edgeIdKey        = null;
  private String                edgeLabelKey     = null;

  /**
   * @param graph
   *          the graph to populate with the GraphML data
   */
  public OGraphMLReader(OrientBaseGraph graph) {
    this.graph = graph;
  }

  /**
   * Input the GraphML stream data into the graph. In practice, usually the provided graph is empty.
   *
   * @param inputGraph
   *          the graph to populate with the GraphML data
   * @param graphMLInputStream
   *          an InputStream of GraphML data
   * @throws IOException
   *           thrown when the GraphML data is not correctly formatted
   */
  public void inputGraph(final Graph inputGraph, final InputStream graphMLInputStream) throws IOException {
    inputGraph(inputGraph, graphMLInputStream, 1000, null, null, null);
  }

  /**
   * Input the GraphML stream data into the graph. In practice, usually the provided graph is empty.
   *
   * @param inputGraph
   *          the graph to populate with the GraphML data
   * @param filename
   *          name of a file containing GraphML data
   * @throws IOException
   *           thrown when the GraphML data is not correctly formatted
   */
  public void inputGraph(final Graph inputGraph, final String filename) throws IOException {
    inputGraph(inputGraph, filename, 1000, null, null, null);
  }

  /**
   * Input the GraphML stream data into the graph. More control over how data is streamed is provided by this method.
   *
   * @param inputGraph
   *          the graph to populate with the GraphML data
   * @param filename
   *          name of a file containing GraphML data
   * @param bufferSize
   *          the amount of elements to hold in memory before committing a transactions (only valid for TransactionalGraphs)
   * @param vertexIdKey
   *          if the id of a vertex is a &lt;data/&gt; property, fetch it from the data property.
   * @param edgeIdKey
   *          if the id of an edge is a &lt;data/&gt; property, fetch it from the data property.
   * @param edgeLabelKey
   *          if the label of an edge is a &lt;data/&gt; property, fetch it from the data property.
   * @throws IOException
   *           thrown when the GraphML data is not correctly formatted
   */
  public void inputGraph(final Graph inputGraph, final String filename, int bufferSize, String vertexIdKey, String edgeIdKey,
      String edgeLabelKey) throws IOException {
    FileInputStream fis = new FileInputStream(filename);
    try {
      inputGraph(inputGraph, fis, bufferSize, vertexIdKey, edgeIdKey, edgeLabelKey);
    } finally {
      fis.close();
    }
  }

  /**
   * Input the GraphML stream data into the graph. More control over how data is streamed is provided by this method.
   *
   * @param inputGraph
   *          the graph to populate with the GraphML data
   * @param graphMLInputStream
   *          an InputStream of GraphML data
   * @param bufferSize
   *          the amount of elements to hold in memory before committing a transactions (only valid for TransactionalGraphs)
   * @param vertexIdKey
   *          if the id of a vertex is a &lt;data/&gt; property, fetch it from the data property.
   * @param edgeIdKey
   *          if the id of an edge is a &lt;data/&gt; property, fetch it from the data property.
   * @param edgeLabelKey
   *          if the label of an edge is a &lt;data/&gt; property, fetch it from the data property.
   * @throws IOException
   *           thrown when the GraphML data is not correctly formatted
   */
  public void inputGraph(final Graph inputGraph, final InputStream graphMLInputStream, int bufferSize, String vertexIdKey,
      String edgeIdKey, String edgeLabelKey) throws IOException {

    XMLInputFactory inputFactory = XMLInputFactory.newInstance();

    try {
      XMLStreamReader reader = inputFactory.createXMLStreamReader(graphMLInputStream);

      final OrientBaseGraph graph = (OrientBaseGraph) inputGraph;

      Map<String, String> keyIdMap = new HashMap<String, String>();
      Map<String, String> keyTypesMaps = new HashMap<String, String>();
      // <Mapped ID String, ID Object>

      // <Default ID String, Mapped ID String>
      Map<String, String> vertexMappedIdMap = new HashMap<String, String>();

      // Buffered Vertex Data
      String vertexId = null;
      Map<String, Object> vertexProps = null;
      boolean inVertex = false;

      // Buffered Edge Data
      String edgeId = null;
      String edgeLabel = null;
      String vertexLabel = null;
      Vertex[] edgeEndVertices = null; // [0] = outVertex , [1] = inVertex
      Map<String, Object> edgeProps = null;
      boolean inEdge = false;

      int bufferCounter = 0;

      while (reader.hasNext()) {

        Integer eventType = reader.next();
        if (eventType.equals(XMLEvent.START_ELEMENT)) {
          String elementName = reader.getName().getLocalPart();

          if (elementName.equals(GraphMLTokens.KEY)) {
            String id = reader.getAttributeValue(null, GraphMLTokens.ID);
            String attributeName = reader.getAttributeValue(null, GraphMLTokens.ATTR_NAME);
            String attributeType = reader.getAttributeValue(null, GraphMLTokens.ATTR_TYPE);
            keyIdMap.put(id, attributeName);
            keyTypesMaps.put(id, attributeType);

          } else if (elementName.equals(GraphMLTokens.NODE)) {
            vertexId = reader.getAttributeValue(null, GraphMLTokens.ID);
            vertexLabel = reader.getAttributeValue(null, LABELS);
            if (vertexLabel != null) {
              if (vertexLabel.startsWith(":"))
                // REMOVE : AS PREFIX
                vertexLabel = vertexLabel.substring(1);

              final String[] vertexLabels = vertexLabel.split(":");

              // GET ONLY FIRST LABEL AS CLASS
              vertexLabel = "class:" + vertexLabels[vertexLabelIndex];
            }
            if (vertexIdKey != null)
              vertexMappedIdMap.put(vertexId, vertexId);
            inVertex = true;
{'fraction_non_alphanumeric': 0.08234342931774365, 'fraction_numerical': 0.0028608744324895826, 'mean_word_length': 2.792452830188679, 'pattern_counts': {'":': 3, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 4, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1894799', 'n_tokens_mistral': 4439, 'n_tokens_neox': 4343, 'n_words': 1521}