<commit_before>package org.col.admin.importer;

import org.col.admin.importer.neo.NeoDb;
import org.col.admin.importer.neo.NodeBatchProcessor;
import org.col.admin.importer.neo.model.Labels;
import org.col.admin.importer.neo.model.NeoName;
import org.col.admin.importer.neo.model.NeoProperties;
import org.col.admin.importer.neo.model.NeoUsage;
import org.col.api.model.VerbatimRecord;
import org.col.api.vocab.Issue;
import org.gbif.dwc.terms.Term;
import org.neo4j.graphdb.Node;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 */
public abstract class RelationInserterBase implements NodeBatchProcessor {
  private static final Logger LOG = LoggerFactory.getLogger(RelationInserterBase.class);
  
  protected final NeoDb store;
  private final Term acceptedTerm;
  private final Term parentTerm;
  
  public RelationInserterBase(NeoDb store, Term acceptedTerm, Term parentTerm) {
    this.store = store;
    this.acceptedTerm = acceptedTerm;
    this.parentTerm = parentTerm;
  }
  
  @Override
  public void process(Node n) {
    if (n.hasLabel(Labels.USAGE)) {
      try {
        NeoUsage u = store.usages().objByNode(n);
        if (u.getVerbatimKey() != null) {
          VerbatimRecord v = store.getVerbatim(u.getVerbatimKey());
          Node p;
          if (u.isSynonym()) {
            p = usageByID(acceptedTerm, v, u);
            if (p != null) {
              store.createSynonymRel(u.node, p);
            } else {
              // if we ain't got no idea of the accepted flag it
              // the orphan synonym usage will be removed later by the normalizer
              v.addIssues(Issue.ACCEPTED_ID_INVALID, Issue.ACCEPTED_NAME_MISSING);
            }
            
          } else {
            p = usageByID(parentTerm, v, u);
            if (p != null) {
              store.assignParent(p, u.node);
            } else {
              v.addIssue(Issue.PARENT_ID_INVALID);
            }
          }
          
          processVerbatimUsage(u, v, p);
          
          store.put(v);
        }
      } catch (Exception e) {
        LOG.error("error processing explicit relations for usage {} {}", n, NeoProperties.getRankedUsage(n), e);
      }
    } else if (n.hasLabel(Labels.NAME)){

      try {
        NeoName nn = store.names().objByNode(n);
        if (nn.getVerbatimKey() != null) {
          VerbatimRecord v = store.getVerbatim(nn.getVerbatimKey());
          processVerbatimName(nn, v);
          store.put(v);
        }
      } catch (Exception e) {
        LOG.error("error processing explicit relations for name {} {}", n, NeoProperties.getScientificNameWithAuthor(n), e);
      }
      
    }
  }
  
  /**
   * @param u
   * @param v
   * @param p parent (usage=taxon) or accepted (usage=synonym) node
   */
  protected void processVerbatimUsage(NeoUsage u, VerbatimRecord v, Node p) {
    // override to do further processing per usage node
  }
  
  /**
   * @param n
   * @param v
   */
  protected void processVerbatimName(NeoName n, VerbatimRecord v) {
    // override to do further processing per name node
  }

  /**
   * Reads a verbatim given term that should represent a foreign key to another record via the taxonID.
   * If the value is not the same as the original records taxonID it tries to split the ids into multiple keys and lookup the matching nodes.
   *
   * @return list of potentially split ids with their matching neo node if found, otherwise null
   */
  protected Node usageByID(Term term, VerbatimRecord v, NeoUsage u) {
    Node n = null;
    final String id = v.get(term);
    if (id != null && !id.equals(u.getId())) {
      n = store.usages().nodeByID(id);
    }
    return n;
  }
  <commit_msg>Yet another bad use of looking up id values not using raw<commit_after>package org.col.admin.importer;

import org.col.admin.importer.neo.NeoDb;
import org.col.admin.importer.neo.NodeBatchProcessor;
import org.col.admin.importer.neo.model.Labels;
import org.col.admin.importer.neo.model.NeoName;
import org.col.admin.importer.neo.model.NeoProperties;
import org.col.admin.importer.neo.model.NeoUsage;
import org.col.api.model.VerbatimRecord;
import org.col.api.vocab.Issue;
import org.gbif.dwc.terms.Term;
import org.neo4j.graphdb.Node;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 *
 */
public abstract class RelationInserterBase implements NodeBatchProcessor {
  private static final Logger LOG = LoggerFactory.getLogger(RelationInserterBase.class);
  
  protected final NeoDb store;
  private final Term acceptedTerm;
  private final Term parentTerm;
  
  public RelationInserterBase(NeoDb store, Term acceptedTerm, Term parentTerm) {
    this.store = store;
    this.acceptedTerm = acceptedTerm;
    this.parentTerm = parentTerm;
  }
  
  @Override
  public void process(Node n) {
    if (n.hasLabel(Labels.USAGE)) {
      try {
        NeoUsage u = store.usages().objByNode(n);
        if (u.getVerbatimKey() != null) {
          VerbatimRecord v = store.getVerbatim(u.getVerbatimKey());
          Node p;
          if (u.isSynonym()) {
            p = usageByID(acceptedTerm, v, u);
            if (p != null) {
              store.createSynonymRel(u.node, p);
            } else {
              // if we ain't got no idea of the accepted flag it
              // the orphan synonym usage will be removed later by the normalizer
              v.addIssues(Issue.ACCEPTED_ID_INVALID, Issue.ACCEPTED_NAME_MISSING);
            }
            
          } else {
            p = usageByID(parentTerm, v, u);
            if (p != null) {
              store.assignParent(p, u.node);
            } else {
              v.addIssue(Issue.PARENT_ID_INVALID);
            }
          }
          
          processVerbatimUsage(u, v, p);
          
          store.put(v);
        }
      } catch (Exception e) {
        LOG.error("error processing explicit relations for usage {} {}", n, NeoProperties.getRankedUsage(n), e);
      }
    } else if (n.hasLabel(Labels.NAME)){

      try {
        NeoName nn = store.names().objByNode(n);
        if (nn.getVerbatimKey() != null) {
          VerbatimRecord v = store.getVerbatim(nn.getVerbatimKey());
          processVerbatimName(nn, v);
          store.put(v);
        }
      } catch (Exception e) {
        LOG.error("error processing explicit relations for name {} {}", n, NeoProperties.getScientificNameWithAuthor(n), e);
      }
      
    }
  }
  
  /**
   * @param u
   * @param v
   * @param p parent (usage=taxon) or accepted (usage=synonym) node
   */
  protected void processVerbatimUsage(NeoUsage u, VerbatimRecord v, Node p) {
    // override to do further processing per usage node
  }
  
  /**
   * @param n
   * @param v
   */
  protected void processVerbatimName(NeoName n, VerbatimRecord v) {
    // override to do further processing per name node
  }

  /**
   * Reads a verbatim given term that should represent a foreign key to another record via the taxonID.
   * If the value is not the same as the original records taxonID it tries to split the ids into multiple keys and lookup the matching nodes.
   *
   * @return list of potentially split ids with their matching neo node if found, otherwise null
   */
  protected Node usageByID(Term idTerm, VerbatimRecord v, NeoUsage u) {
    Node n = null;
    final String id = v.getRaw(idTerm);
    if (id != null && !id.equals(u.getId())) {
      n = store.usages().nodeByID(id);
    }
    return n;
  }
  
{'fraction_non_alphanumeric': 0.10588710763842113, 'fraction_numerical': 0.000808298531591001, 'mean_word_length': 2.4338575393154485, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5713641', 'n_tokens_mistral': 2354, 'n_tokens_neox': 2286, 'n_words': 613}