<commit_before>/*
 *    Copyright 2015 West Coast Informatics, LLC
 */
package com.wci.umls.server.jpa.algo;

import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;

import com.wci.umls.server.Project;
import com.wci.umls.server.ValidationResult;
import com.wci.umls.server.helpers.Branch;
import com.wci.umls.server.helpers.FieldedStringTokenizer;
import com.wci.umls.server.helpers.QueryType;
import com.wci.umls.server.jpa.algo.action.AddDemotionMolecularAction;
import com.wci.umls.server.jpa.algo.action.MergeMolecularAction;
import com.wci.umls.server.jpa.content.AtomJpa;
import com.wci.umls.server.jpa.content.ConceptJpa;
import com.wci.umls.server.model.content.Atom;
import com.wci.umls.server.model.content.Concept;
import com.wci.umls.server.model.content.ConceptRelationship;

/**
 * Abstract support for merge algorithms.
 */
public abstract class AbstractMergeAlgorithm
    extends AbstractInsertMaintReleaseAlgorithm {

  /**
   * Instantiates an empty {@link AbstractMergeAlgorithm}.
   *
   * @throws Exception the exception
   */
  public AbstractMergeAlgorithm() throws Exception {
    // n/a
  }

  /**
   * Merge.
   *
   * @param atomId the atom id
   * @param atomId2 the atom id 2
   * @param validationChecks the integrity check names
   * @param makeDemotion the make demotion
   * @param changeStatus the change status
   * @param project the project
   * @param statsMap the stats map
   * @throws Exception the exception
   */
  public void merge(Long atomId, Long atomId2, List<String> validationChecks,
    boolean makeDemotion, boolean changeStatus, Project project,
    Map<String, Integer> statsMap) throws Exception {

    // Get the two concepts associated with the two atoms
    List<ConceptJpa> concepts =
        searchHandler.getQueryResults(getProject().getTerminology(),
            getProject().getVersion(), Branch.ROOT, "atoms.id:" + atomId, null,
            ConceptJpa.class, null, new int[1], getEntityManager());
    if (concepts.size() != 1) {
      throw new Exception("Unexpected number of concepts: " + concepts.size()
          + ", for atom: " + atomId);
    }
    final Concept concept = concepts.get(0);

    concepts = searchHandler.getQueryResults(getProject().getTerminology(),
        getProject().getVersion(), Branch.ROOT, "atoms.id:" + atomId2, null,
        ConceptJpa.class, null, new int[1], getEntityManager());
    if (concepts.size() != 1) {
      throw new Exception("Unexpected number of concepts: " + concepts.size()
          + ", for atom: " + atomId2);
    }
    final Concept concept2 = concepts.get(0);

    // Identify the from and to concepts, and from/to Atoms
    // FromConcept will be the smaller concept (least number of atoms)
    Concept fromConcept = null;
    Concept toConcept = null;
    Atom fromAtom = null;
    Atom toAtom = null;

    if (concept.getAtoms().size() < concept2.getAtoms().size()) {
      fromConcept = concept;
      fromAtom = getAtom(atomId);
      toConcept = concept2;
      toAtom = getAtom(atomId2);
    } else {
      fromConcept = concept2;
      fromAtom = getAtom(atomId2);
      toConcept = concept;
      toAtom = getAtom(atomId);
    }

    // If Atoms are in the same concept, DON'T perform merge, and log that the
    // atoms are already merged.
    if (fromConcept.getId().equals(toConcept.getId())) {
      addLogEntry(getLastModifiedBy(), getProject().getId(),
          fromConcept.getId(), getActivityId(), getWorkId(),
          "Failure merging atom " + atomId + " with atom " + atomId2
              + " - atoms are both already in the same concept "
              + toConcept.getId());

      statsMap.put("unsuccessfulMerges",
          statsMap.get("unsuccessfulMerges") + 1);
      return;
    }

    // Otherwise, create and set up a merge action
    final MergeMolecularAction action = new MergeMolecularAction();

    try {

      // Configure the action
      action.setProject(getProject());
      action.setTerminology(getProject().getTerminology());
      action.setVersion(getProject().getVersion());
      action.setWorkId(getWorkId());
      action.setActivityId(getActivityId());
      action.setConceptId(fromConcept.getId());
      action.setConceptId2(toConcept.getId());
      action.setLastModifiedBy(getLastModifiedBy());
      action.setLastModified(fromConcept.getLastModified().getTime());<commit_msg>NE-289: AbstractMergeAlgorithm now keeps track of concept-pairs, and only attempts to merge them one time.<commit_after>/*
 *    Copyright 2015 West Coast Informatics, LLC
 */
package com.wci.umls.server.jpa.algo;

import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;

import com.wci.umls.server.Project;
import com.wci.umls.server.ValidationResult;
import com.wci.umls.server.helpers.Branch;
import com.wci.umls.server.helpers.FieldedStringTokenizer;
import com.wci.umls.server.helpers.QueryType;
import com.wci.umls.server.jpa.algo.action.AddDemotionMolecularAction;
import com.wci.umls.server.jpa.algo.action.MergeMolecularAction;
import com.wci.umls.server.jpa.content.AtomJpa;
import com.wci.umls.server.jpa.content.ConceptJpa;
import com.wci.umls.server.model.content.Atom;
import com.wci.umls.server.model.content.Concept;
import com.wci.umls.server.model.content.ConceptRelationship;

/**
 * Abstract support for merge algorithms.
 */
public abstract class AbstractMergeAlgorithm
    extends AbstractInsertMaintReleaseAlgorithm {

  /**
   * The concept-pairs that have already gone through the merging process
   */
  private Set<String> conceptsAlreadyMerged = new HashSet<>();

  /**
   * Instantiates an empty {@link AbstractMergeAlgorithm}.
   *
   * @throws Exception the exception
   */
  public AbstractMergeAlgorithm() throws Exception {
    // n/a
  }

  /**
   * Merge.
   *
   * @param atomId the atom id
   * @param atomId2 the atom id 2
   * @param validationChecks the integrity check names
   * @param makeDemotion the make demotion
   * @param changeStatus the change status
   * @param project the project
   * @param statsMap the stats map
   * @throws Exception the exception
   */
  public void merge(Long atomId, Long atomId2, List<String> validationChecks,
    boolean makeDemotion, boolean changeStatus, Project project,
    Map<String, Integer> statsMap) throws Exception {

    // Get the two concepts associated with the two atoms
    List<ConceptJpa> concepts =
        searchHandler.getQueryResults(getProject().getTerminology(),
            getProject().getVersion(), Branch.ROOT, "atoms.id:" + atomId, null,
            ConceptJpa.class, null, new int[1], getEntityManager());
    if (concepts.size() != 1) {
      throw new Exception("Unexpected number of concepts: " + concepts.size()
          + ", for atom: " + atomId);
    }
    final Concept concept = concepts.get(0);

    concepts = searchHandler.getQueryResults(getProject().getTerminology(),
        getProject().getVersion(), Branch.ROOT, "atoms.id:" + atomId2, null,
        ConceptJpa.class, null, new int[1], getEntityManager());
    if (concepts.size() != 1) {
      throw new Exception("Unexpected number of concepts: " + concepts.size()
          + ", for atom: " + atomId2);
    }
    final Concept concept2 = concepts.get(0);

    // Identify the from and to concepts, and from/to Atoms
    // FromConcept will be the smaller concept (least number of atoms)
    Concept fromConcept = null;
    Concept toConcept = null;
    Atom fromAtom = null;
    Atom toAtom = null;

    if (concept.getAtoms().size() < concept2.getAtoms().size()) {
      fromConcept = concept;
      fromAtom = getAtom(atomId);
      toConcept = concept2;
      toAtom = getAtom(atomId2);
    } else {
      fromConcept = concept2;
      fromAtom = getAtom(atomId2);
      toConcept = concept;
      toAtom = getAtom(atomId);
    }

    // If this concept pair has already had a merge attempted on it, don't try
    // it again
    if (conceptsAlreadyMerged
        .contains(fromConcept.getId() + "|" + toConcept.getId())) {
      return;
    } else {
      conceptsAlreadyMerged.add(fromConcept.getId() + "|" + toConcept.getId());
    }

    // If Atoms are in the same concept, DON'T perform merge, and log that the
    // atoms are already merged.
    if (fromConcept.getId().equals(toConcept.getId())) {
      addLogEntry(getLastModifiedBy(), getProject().getId(),
          fromConcept.getId(), getActivityId(), getWorkId(),
          "Failure merging atom " + atomId + " with atom " + atomId2
              + " - atoms are both already in the same concept "
              + toConcept.getId());

      statsMap.put("unsuccessfulMerges",
          statsMap.get("unsuccessfulMerges") + 1);
      return;
    }

    // Otherwise, create and set up a merge action
    final MergeMolecularAction action = new MergeMolecularAction();

    try {

      // Configure the action
      action.setProject(getProject());
      action.setTerminology(getProject().getTerminology());
      action.setVersion(getProject().getVersion());
      action.setWorkId(getWorkId());
      action.setActivityId(getActivityId());
      action.setConceptId(fromConcept.getId());
      action.setConceptId2(toConcept.getId());
      action.setLastModifiedBy(getLastModifiedBy());
      action.setLastModified(fromConcept.getLastModified().getTime());
{'fraction_non_alphanumeric': 0.11596263200649878, 'fraction_numerical': 0.005787977254264825, 'mean_word_length': 3.3812277580071175, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '379946', 'n_tokens_mistral': 3032, 'n_tokens_neox': 2991, 'n_words': 712}