<commit_before>/*
 * Copyright 2016 <NAME>.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.corpus_tools.graphannis;

import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.corpus_tools.salt.SALT_TYPE;
import org.corpus_tools.salt.common.SDocumentGraph;
import org.corpus_tools.salt.common.SDominanceRelation;
import org.corpus_tools.salt.common.SPointingRelation;
import org.corpus_tools.salt.common.SStructure;
import org.corpus_tools.salt.common.SStructuredNode;
import org.corpus_tools.salt.common.STextualDS;
import org.corpus_tools.salt.common.STextualRelation;
import org.corpus_tools.salt.common.SToken;
import org.corpus_tools.salt.core.SAnnotation;
import org.corpus_tools.salt.core.SLayer;
import org.corpus_tools.salt.core.SNode;
import org.corpus_tools.salt.core.SRelation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A class which helps to import salt documents into graphANNIS.
 *
 * @author <NAME> <<EMAIL>>
 */
public class SaltImport
{

  public static final String ANNIS_NS = "annis4_internal";

  private static final Logger log = LoggerFactory.getLogger(SaltImport.class);

  private final API.GraphUpdate updateList = new API.GraphUpdate();
  
  public API.GraphUpdate finish()
  {
    return updateList;
  }
  
  
  public SaltImport map(SDocumentGraph g)
  {
    API.GraphUpdate u = new API.GraphUpdate();

    // add all nodes and their annotations
    for (SNode n : g.getNodes())
    {
      addNode(n);
    }

    Map<SToken, Long> token2Index = addTokenInformation(g);
    
    for(SNode n : g.getNodes())
    {
      addLeftRightToken(n, g, token2Index);
    }
      
    for (SDominanceRelation rel : g.getDominanceRelations())
    {
      String sourceName = nodeName(rel.getSource());
      String targetName = nodeName(rel.getTarget());

      for (String l : getLayerNames(rel.getLayers()))
      {
        // add an edge both for the named component and for the un-named
        u.addEdge(sourceName, targetName, l, "DOMINANCE", rel.getType());
        u.addEdge(sourceName, targetName, l, "DOMINANCE", "");
      }
    }

    for (SPointingRelation rel : g.getPointingRelations())
    {
      String sourceName = nodeName(rel.getSource());
      String targetName = nodeName(rel.getTarget());

      for (String l : getLayerNames(rel.getLayers()))
      {
        // add an edge both for the named component and for the un-named
        u.addEdge(sourceName, targetName, l, "POINTING", rel.getType());
      }
    }

    return this;
  }

  private Map<SToken, Long> addTokenInformation(SDocumentGraph g)
  {
    Map<SToken, Long> token2Index = new HashMap<>();

    SToken lastToken = null;
    STextualDS lastTextDS = null;

    List<SToken> sortedToken = g.getSortedTokenByText();

    if (sortedToken != null)
    {
      long i = 0;
      for (SToken t : sortedToken)
      {
        token2Index.put(t, i++);

        String nodeName = nodeName(t);
        // each token must have it's spanned text as label
        updateList.addNodeLabel(nodeName, ANNIS_NS, "tok", g.getText(t));

        STextualDS textDS = getTextForToken(t);
        if (lastToken != null && textDS == lastTextDS)
        {
          // add an explicit ORDERING edge between the token
          updateList.addEdge(nodeName(lastToken), nodeName, ANNIS_NS, "ORDERING", "");
        }
        lastToken = t;
        lastTextDS = textDS;
      }
    }
    return token2Index;
  }

  /**
   * Add an explicit edge to the left- and right-most covered token of the node.
   *
   * @param node
   * @param graph
   * @param token2Index
   */
  private void addLeftRightToken(SNode node, SDocumentGraph graph, Map<SToken, Long> token2Index)
  {
    List<SToken> overlappedToken;
    if (node instanceof SToken)
    {
      overlappedToken = Arrays.asList((SToken) node);
    }
    else if (node instanceof SStructure)
    {
      overlappedToken = graph.getOverlappedTokens(node, SALT_TYPE.SSPANNING_RELATION,
        SALT_TYPE.SDOMINANCE_RELATION);
    }
    else
    {
      overlappedToken = graph.getOverlappedTokens(node, SALT_TYPE.SSPANNING_RELATION);

    }
    if (overlappedToken.isEmpty())
    {
      log.warn("Node {} is not connected to any token. This is invalid for graphANNIS and the node will be excluded.", node.getId());
      return;<commit_msg>Helper index is not needed.<commit_after>/*
 * Copyright 2016 <NAME>.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.corpus_tools.graphannis;

import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.corpus_tools.salt.SALT_TYPE;
import org.corpus_tools.salt.common.SDocumentGraph;
import org.corpus_tools.salt.common.SDominanceRelation;
import org.corpus_tools.salt.common.SPointingRelation;
import org.corpus_tools.salt.common.SStructure;
import org.corpus_tools.salt.common.SStructuredNode;
import org.corpus_tools.salt.common.STextualDS;
import org.corpus_tools.salt.common.STextualRelation;
import org.corpus_tools.salt.common.SToken;
import org.corpus_tools.salt.core.SAnnotation;
import org.corpus_tools.salt.core.SLayer;
import org.corpus_tools.salt.core.SNode;
import org.corpus_tools.salt.core.SRelation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * A class which helps to import salt documents into graphANNIS.
 *
 * @author <NAME> <<EMAIL>>
 */
public class SaltImport
{

  public static final String ANNIS_NS = "annis4_internal";

  private static final Logger log = LoggerFactory.getLogger(SaltImport.class);

  private final API.GraphUpdate updateList = new API.GraphUpdate();
  
  public API.GraphUpdate finish()
  {
    return updateList;
  }
  
  
  public SaltImport map(SDocumentGraph g)
  {
    API.GraphUpdate u = new API.GraphUpdate();

    // add all nodes and their annotations
    for (SNode n : g.getNodes())
    {
      addNode(n);
    }

    addTokenInformation(g);
    
    for(SNode n : g.getNodes())
    {
      addLeftRightToken(n, g);
    }
      
    for (SDominanceRelation rel : g.getDominanceRelations())
    {
      String sourceName = nodeName(rel.getSource());
      String targetName = nodeName(rel.getTarget());

      for (String l : getLayerNames(rel.getLayers()))
      {
        // add an edge both for the named component and for the un-named
        u.addEdge(sourceName, targetName, l, "DOMINANCE", rel.getType());
        u.addEdge(sourceName, targetName, l, "DOMINANCE", "");
      }
    }

    for (SPointingRelation rel : g.getPointingRelations())
    {
      String sourceName = nodeName(rel.getSource());
      String targetName = nodeName(rel.getTarget());

      for (String l : getLayerNames(rel.getLayers()))
      {
        // add an edge both for the named component and for the un-named
        u.addEdge(sourceName, targetName, l, "POINTING", rel.getType());
      }
    }

    return this;
  }

  private void addTokenInformation(SDocumentGraph g)
  {

    SToken lastToken = null;
    STextualDS lastTextDS = null;

    List<SToken> sortedToken = g.getSortedTokenByText();

    if (sortedToken != null)
    {
      long i = 0;
      for (SToken t : sortedToken)
      {

        String nodeName = nodeName(t);
        // each token must have it's spanned text as label
        updateList.addNodeLabel(nodeName, ANNIS_NS, "tok", g.getText(t));

        STextualDS textDS = getTextForToken(t);
        if (lastToken != null && textDS == lastTextDS)
        {
          // add an explicit ORDERING edge between the token
          updateList.addEdge(nodeName(lastToken), nodeName, ANNIS_NS, "ORDERING", "");
        }
        lastToken = t;
        lastTextDS = textDS;
      }
    }
  }

  /**
   * Add an explicit edge to the left- and right-most covered token of the node.
   *
   * @param node
   * @param graph
   */
  private void addLeftRightToken(SNode node, SDocumentGraph graph)
  {
    List<SToken> overlappedToken;
    if (node instanceof SToken)
    {
      overlappedToken = Arrays.asList((SToken) node);
    }
    else if (node instanceof SStructure)
    {
      overlappedToken = graph.getOverlappedTokens(node, SALT_TYPE.SSPANNING_RELATION,
        SALT_TYPE.SDOMINANCE_RELATION);
    }
    else
    {
      overlappedToken = graph.getOverlappedTokens(node, SALT_TYPE.SSPANNING_RELATION);

    }
    if (overlappedToken.isEmpty())
    {
      log.warn("Node {} is not connected to any token. This is invalid for graphANNIS and the node will be excluded.", node.getId());
      return;
{'fraction_non_alphanumeric': 0.10291524017014213, 'fraction_numerical': 0.0032161012553169415, 'mean_word_length': 3.1444539982803095, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '260184', 'n_tokens_mistral': 3081, 'n_tokens_neox': 2989, 'n_words': 800}