<commit_before>/*
 *    Copyright 2015 West Coast Informatics, LLC
 */
package com.wci.umls.server.jpa.algo.insert;

import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;

import com.wci.umls.server.AlgorithmParameter;
import com.wci.umls.server.ValidationResult;
import com.wci.umls.server.helpers.QueryType;
import com.wci.umls.server.jpa.AlgorithmParameterJpa;
import com.wci.umls.server.jpa.ValidationResultJpa;
import com.wci.umls.server.jpa.algo.AbstractSourceInsertionAlgorithm;
import com.wci.umls.server.model.content.Atom;
import com.wci.umls.server.model.content.AtomRelationship;
import com.wci.umls.server.model.workflow.WorkflowStatus;

/**
 * Implementation of an algorithm to remove demotions from atoms matching
 * criteria.
 */
public class SafeReplaceAlgorithm extends AbstractSourceInsertionAlgorithm {

  /** The string class id flag. */
  private Boolean stringClassId = null;

  /** The lexical class id flag. */
  private Boolean lexicalClassId = null;

  /** The code id flag. */
  private Boolean codeId = null;

  /** The concept id flag. */
  private Boolean conceptId = null;

  /** The descriptor id flag. */
  private Boolean descriptorId = null;

  /**
   * Instantiates an empty {@link SafeReplaceAlgorithm}.
   * @throws Exception if anything goes wrong
   */
  public SafeReplaceAlgorithm() throws Exception {
    super();
    setActivityId(UUID.randomUUID().toString());
    setWorkId("SAFEREPLACE");
    setLastModifiedBy("admin");
  }

  /**
   * Check preconditions.
   *
   * @return the validation result
   * @throws Exception the exception
   */
  /* see superclass */
  @Override
  public ValidationResult checkPreconditions() throws Exception {

    ValidationResult validationResult = new ValidationResultJpa();

    if (getProject() == null) {
      throw new Exception("Safe Replace requires a project to be set");
    }

    return validationResult;
  }

  /**
   * Compute.
   *
   * @throws Exception the exception
   */
  /* see superclass */
  @Override
  public void compute() throws Exception {
    logInfo("Starting SAFEREPLACE");

    // Molecular actions will be generated by this algorithm
    // TODO - molecular actions?
    setMolecularActionFlag(true);

    // Find all atoms pairs that satisfy the specified matching criteria and are
    // in the same [project Terminology] concept

    // Generate query string
     String query =
        "SELECT a1.id, a2.id "
            + "FROM ConceptJpa c1 JOIN c1.atoms a1, ConceptJpa c2 JOIN c2.atoms a2 "
            + "WHERE c1.id = c2.id " + "AND NOT a1.id = a2.id "
            + "AND c1.terminology=:projectTerminology AND c1.version=:projectVersion "
            + (stringClassId ? "AND a1.stringClassId = a2.stringClassId " : "")
            + (lexicalClassId ? "AND a1.lexicalClassId = a2.lexicalClassId "
                : "")
            + (conceptId ? "AND a1.conceptId = a2.conceptId " : "")
            + (codeId ? "AND a1.codeId = a2.codeId " : "")
            + (descriptorId ? "AND a1.descriptorId = a2.descriptorId " : "");

    // Generate parameters to pass into query executions
    Map<String, String> params = new HashMap<>();
    params.put("projectTerminology", getProject().getTerminology());
    params.put("projectVersion", getProject().getVersion());

    final List<Long[]> atomIdPairArray =
        executeAtomIdPairQuery(query, QueryType.JQL, params);

    // Create map <Long,Set<Long>)
    // Key=fromAtomId,
    // Value=set of all toAtomIds associated with fromAtomId
    final Map<Long, Set<Long>> atomPairMap = new HashMap<>();
    for (Long[] atomIdPair : atomIdPairArray) {
      Set<Long> associatedAtoms = atomPairMap.get(atomIdPair[0]);
      if (associatedAtoms == null || associatedAtoms.isEmpty()) {
        associatedAtoms = new HashSet<>();
      }
      associatedAtoms.add(atomIdPair[1]);
      atomPairMap.put(atomIdPair[0], associatedAtoms);
    }

    setSteps(atomPairMap.size());
    
    logInfo("[Safe Replace] removing demotions");

    // If an atom has a demotion to any of its associated atoms, remove it
    for (Map.Entry<Long, Set<Long>> entry : atomPairMap.entrySet()) {
      Long atomId = entry.getKey();
      Set<Long> associatedAtomIds = entry.getValue();

      final Atom atom = getAtom(atomId);
      final Set<Long> removeRelationshipIds = new HashSet<>();

      for (AtomRelationship rel : atom.getRelationships()) {
        if (rel.getWorkflowStatus().equals(WorkflowStatus.DEMOTION)
            && associatedAtomIds.contains(rel.getTo().getId())) {

          // TODO - if molecular actions, create removeDemotionMolecularAction?
          // Or find addDemotion, and undo it (seems overkill)

          // Remove the demotion from the atom
          atom.getRelationships().remove(rel);

          // Save the relationship Id, to remove later
          removeRelationshipIds.add(rel.getId());
        }
      }

      // If any demotions are removed from the atom, update the atom, and
      // delete the demotions
      if (removeRelationshipIds.size() > 0) {
        updateAtom(atom);
      }
      // And delete the demotions
      for (Long relId : removeRelationshipIds) {
        removeRelationship(relId, AtomRelationship.class);
        // Note: Don't need to handle the inverse demotion - the reverse
        // atomPair will also be included in the query results
      }<commit_msg>NE-160: Modified SafeReplaceAlgorithm based on updated ticket.<commit_after>/*
 *    Copyright 2015 West Coast Informatics, LLC
 */
package com.wci.umls.server.jpa.algo.insert;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.UUID;

import org.apache.commons.lang3.tuple.ImmutablePair;
import org.apache.commons.lang3.tuple.Pair;

import com.wci.umls.server.AlgorithmParameter;
import com.wci.umls.server.ValidationResult;
import com.wci.umls.server.helpers.QueryType;
import com.wci.umls.server.jpa.AlgorithmParameterJpa;
import com.wci.umls.server.jpa.ValidationResultJpa;
import com.wci.umls.server.jpa.algo.AbstractMergeAlgorithm;
import com.wci.umls.server.model.content.Atom;
import com.wci.umls.server.model.content.AtomRelationship;
import com.wci.umls.server.model.meta.TermType;
import com.wci.umls.server.model.workflow.WorkflowStatus;

/**
 * Implementation of an algorithm to remove demotions from atoms matching
 * criteria.
 */
public class SafeReplaceAlgorithm extends AbstractMergeAlgorithm {

  /** The string class id flag. */
  private Boolean stringClassId = null;

  /** The lexical class id flag. */
  private Boolean lexicalClassId = null;

  /** The code id flag. */
  private Boolean codeId = null;

  /** The concept id flag. */
  private Boolean conceptId = null;

  /** The descriptor id flag. */
  private Boolean descriptorId = null;

  /**
   * Instantiates an empty {@link SafeReplaceAlgorithm}.
   * @throws Exception if anything goes wrong
   */
  public SafeReplaceAlgorithm() throws Exception {
    super();
    setActivityId(UUID.randomUUID().toString());
    setWorkId("SAFEREPLACE");
    setLastModifiedBy("admin");
  }

  /**
   * Check preconditions.
   *
   * @return the validation result
   * @throws Exception the exception
   */
  /* see superclass */
  @Override
  public ValidationResult checkPreconditions() throws Exception {

    ValidationResult validationResult = new ValidationResultJpa();

    if (getProject() == null) {
      throw new Exception("Safe Replace requires a project to be set");
    }

    // TODO - is this true?
    if(!(codeId || conceptId || descriptorId || lexicalClassId || stringClassId)){
      validationResult.addWarning("WARNING: no match-criteria are selected (e.g. code Id, concept Id, etc.).");
    }
    
    return validationResult;
  }

  /**
   * Compute.
   *
   * @throws Exception the exception
   */
  /* see superclass */
  @Override
  public void compute() throws Exception {
    logInfo("Starting SAFEREPLACE");

    // Not Molecular actions will be generated by this algorithm
    setMolecularActionFlag(false);

    // Find all atoms pairs that satisfy the specified matching criteria and are
    // in the same project Terminology concept, where one atom is old and the
    // other is new

    // Generate query string
    String query =
        "SELECT DISTINCT a1.id, a2.id "
            + "FROM ConceptJpa c JOIN c.atoms a1 JOIN c.atoms a2 "
            + "WHERE NOT a1.id = a2.id "
            + "AND c.terminology=:projectTerminology AND c.version=:projectVersion "
            + "AND a1.terminology=:terminology AND NOT a1.version=:version "
            + "AND a2.terminology=:terminology AND a2.version=:version "
            + (stringClassId ? "AND a1.stringClassId = a2.stringClassId " : "")
            + (lexicalClassId ? "AND a1.lexicalClassId = a2.lexicalClassId "
                : "")
            + (conceptId ? "AND a1.conceptId = a2.conceptId " : "")
            + (codeId ? "AND a1.codeId = a2.codeId " : "")
            + (descriptorId ? "AND a1.descriptorId = a2.descriptorId " : "");

    // Generate parameters to pass into query executions
    Map<String, String> params = new HashMap<>();
    params.put("projectTerminology", getProject().getTerminology());
    params.put("projectVersion", getProject().getVersion());
    params.put("terminology", this.getTerminology());
    params.put("version", this.getVersion());

    final List<Long[]> atomIdPairArray =
        executeAtomIdPairQuery(query, QueryType.JQL, params);

    setSteps(atomIdPairArray.size());

    // Recast the array as a Pair, for easier comparison
    final List<Pair<Long, Long>> atomIdPairs = new ArrayList<>();
    for (Long[] atomidPair : atomIdPairArray) {
      atomIdPairs
          .add(new ImmutablePair<Long, Long>(atomidPair[0], atomidPair[1]));
    }

    // Sort pairs by MergeLevel, AtomId1, and AtomId2
    sortPairsByMergeLevelAndId(atomIdPairs);

    logInfo("[Safe Replace] performing safe replacements");

    // Track which new atoms have been safe-replaced:
    // Each new atom can only be safe-replaced by a single old-atom.
    Set<Long> safeReplacedAtomIds = new HashSet<>();

    for (Pair<Long, Long> atomIdPair : atomIdPairs) {
      final Long oldAtomId = atomIdPair.getLeft();
      final Long newAtomId = atomIdPair.getRight();

      // If this newAtom has already had a safe-replacement performed on it,
      // skip
      if (safeReplacedAtomIds.contains(newAtomId)) {
        continue;
      } else {
        safeReplacedAtomIds.add(newAtomId);
      }

      // Borrow information from the old atom and assign to the new atom
      final Atom oldAtom = getAtom(oldAtomId);
      final Atom newAtom = getAtom(newAtomId);
      
      // Update newAtom's alternateTerminologyIds
      for(final Map.Entry<String, String> oldAltTermId : oldAtom.getAlternateTerminologyIds().entrySet()){
        newAtom.getAlternateTerminologyIds().put(oldAltTermId.getKey(), oldAltTermId.getValue());
      }

      // TODO - do equivalence check for these atoms?
      // If the old Atom is not exactly the same as the new Atom:
      if (!oldAtom.equals(newAtom)) {

        //TODO question - figure out new/old ordering of things
        // Update obsolete and suppresible.
        // If the old version of the atom is suppresible, and its term type
        // is not, keep the old atom's suppresibility. Otherwise, use the
        // new Atom's suppresible value.
        TermType atomTty = getCachedTermType(oldAtom.getTermType());
        if (oldAtom.isSuppressible() != newAtom.isSuppressible()
            && !(oldAtom.isSuppressible() && !atomTty.isSuppressible())) {
          newAtom.setSuppressible(oldAtom.isSuppressible());
        }
        if (oldAtom.isObsolete() != newAtom.isObsolete()
            && !(oldAtom.isObsolete() && !atomTty.isObsolete())) {
          newAtom.setObsolete(oldAtom.isObsolete());
        }
 
        if (!oldAtom.getWorkflowStatus().equals(newAtom.getWorkflowStatus())){
          newAtom.setWorkflowStatus(oldAtom.getWorkflowStatus());
        }        
        
        //TODO question - releaseRank = lastPublishedRank?
        if (!oldAtom.getLastPublishedRank().equals(newAtom.getLastPublishedRank())){
          newAtom.setLastPublishedRank(oldAtom.getLastPublishedRank());
        }
      }

      // Remove all demotions from the new Atom, and the inverses
      final Set<Long> removeRelationshipIds = new HashSet<>();

      for (AtomRelationship rel : newAtom.getRelationships()) {
        if (rel.getWorkflowStatus().equals(WorkflowStatus.DEMOTION)) {
          // Remove the demotion from the atom
          newAtom.getRelationships().remove(rel);

          // Remove the inverse relationship from the toAtom
          Atom relatedAtom = getAtom(rel.getTo().getId());
          AtomRelationship inverseDemotion =
              (AtomRelationship) findInverseRelationship(rel);
          relatedAtom.getRelationships().remove(inverseDemotion);

          // Update the related atom
          updateAtom(relatedAtom);

          // Save the demotion and inverseDemotion Ids, to delete later
          removeRelationshipIds.add(rel.getId());
          removeRelationshipIds.add(inverseDemotion.getId());
        }
      }

      // Once all demotions are removed, update the newAtom
      updateAtom(newAtom);

      // Delete any demotions that were removed from atoms
      for (Long relId : removeRelationshipIds) {
        removeRelationship(relId, AtomRelationship.class);
      }
      
{'fraction_non_alphanumeric': 0.11057587037442522, 'fraction_numerical': 0.004744179256988541, 'mean_word_length': 2.964699074074074, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1895389', 'n_tokens_mistral': 4123, 'n_tokens_neox': 3986, 'n_words': 1040}